@pyric/cli 0.1.0-alpha.9

package/dist/discover/crawler.d.ts

@@ -0,0 +1,294 @@
+import { SessionStore } from './session.js';
+import type { CrawlerCollectionRef, CrawlerDocumentRef, CrawlerFirestore } from './firestore-source.js';
+import type { CollectionSchema, DiscoverEvent } from './types.js';
+/**
+ * Crawl options. All fields are optional with documented defaults.
+ *
+ * - `maxConcurrency`: in-flight RPC cap. Default 32 per the Risk #1 sweep
+ *   on 2026-05-05. Sweep over `{4, 8, 16, 32, 64}` against the corpus
+ *   showed `4 → 8 → 16 → 32 → 64` was a steady ~10–15% per-doubling
+ *   descent (no plateau). 32 was picked over 64 because the curve hadn't
+ *   flattened — 64 was the cap of the test range, not a true knee — and
+ *   doubling in-flight RPCs again increases the chance of tripping
+ *   per-project connection/quota limits in agent environments. Agents
+ *   that want max speed can override.
+ * - `maxDepth`: hard cap on BFS layers from the root. Defaults to 10
+ *   (well above any real-world Firestore tree). Used as a runaway guard,
+ *   not an agent-facing knob.
+ * - `rootFilter`: optional predicate on root collection IDs. Used by tests
+ *   and the corpus harness to scope discovery to a known prefix without
+ *   walking the entire database.
+ */
+export interface CrawlOptions {
+    maxConcurrency?: number;
+    maxDepth?: number;
+    rootFilter?: (collectionId: string) => boolean;
+    /**
+     * Hard cap on docs sampled per templatePath. Default 50 per Phase 2.1
+     * lock. Item 3 will add adaptive `stopOnStable` early-exit on top of
+     * this cap; for now sampling reads exactly `min(maxSamples, available)`
+     * docs per templatePath.
+     */
+    maxSamples?: number;
+    /**
+     * Per-templatePath cap on tolerated PERMISSION_DENIED / transient errors
+     * during sampling. Default 3 per prerequisite 0.E. Past this threshold,
+     * sampling for the templatePath stops and `samplingComplete` is set to
+     * `sampling_open` so the agent can see the collection wasn't fully
+     * sampled. Errors during structure discovery are emitted but do not
+     * count toward this cap.
+     */
+    maxErrorsPerCollection?: number;
+    /**
+     * Adaptive early-exit threshold. After this many consecutive
+     * no-change merges in a templatePath's sampling stream, sampling stops
+     * and `samplingComplete` is set to `converged_via_stable`. Default 8 per
+     * Phase 2.1 lock. Set to a value > maxSamples to disable early-exit
+     * (the hard cap then governs).
+     *
+     * Reads are issued in chunks of `stopOnStable` so an early-exit avoids
+     * fetching the remainder of `sampleRefs`. Worst-case wasted-read
+     * count per templatePath is `stopOnStable - 1`.
+     */
+    stopOnStable?: number;
+    /**
+     * Resume a previously paused crawl. Only valid when a `SessionStore` is
+     * passed to `crawl()`. The token is the `continuation` value from a
+     * prior paused result. Malformed/expired tokens surface as a
+     * `SESSION_EXPIRED`/`SESSION_EVICTED`/`SESSION_MALFORMED_TOKEN`
+     * error event with no other side effects (per 0.C).
+     */
+    continuation?: string;
+    /**
+     * Pause threshold for batch payload size. After each layer (structure
+     * phase) and each templatePath (sampling phase) the crawler measures
+     * the JSON-serialized state size; if it exceeds this many bytes, the
+     * crawl pauses and returns a continuation token. Default 1 MB per
+     * Phase 0.4 sizing — well below `maxSessionBytes=32MB` so the agent
+     * has headroom for response framing.
+     *
+     * Only effective when a `SessionStore` is provided to `crawl()`. With
+     * no store, single-call mode runs to completion regardless of size.
+     */
+    maxBatchBytes?: number;
+    /**
+     * Informational-only cost preview. When `true`, the crawler issues
+     * exactly **one** RPC — `db.listCollections()` at the root — and
+     * returns a heuristic projection of what a real crawl would cost.
+     *
+     * No documents are read, no per-doc `listCollections` calls are
+     * made, no sampling occurs. The result has empty `discovered` and
+     * `finalizedSchemas`, `complete: true`, and `continuation: undefined`.
+     * It is **not** a partial crawl that can be resumed — to "commit",
+     * call `crawl()` again with `dryRun: false` (a fresh full crawl).
+     *
+     * Rationale: an agent reading `dryRun: true` reasonably expects no
+     * real crawl happened. Doing a structure walk under that flag would
+     * let the agent make decisions on data it didn't realize it paid
+     * for. See Item 5 revision in the implementation plan.
+     *
+     * Default `false`.
+     */
+    dryRun?: boolean;
+    /**
+     * Heuristic multiplier used in the dryRun cost projection: assumes
+     * each root collection has roughly this many subtree-collections
+     * (root + descendants) on average. Default 3 — conservative for
+     * typical app schemas. Surfaced as an option so agents tuning for
+     * known shapes can revise.
+     *
+     * Only consulted when `dryRun: true`.
+     */
+    dryRunSubtreeMultiplier?: number;
+}
+/**
+ * Per-template-path bookkeeping built up during a crawl. Collection refs
+ * are kept here for Item 2.3 to drive document sampling.
+ *
+ * Multiple concrete collection paths may collapse to the same template
+ * path (e.g. `users/uid_1/posts` and `users/uid_2/posts` both map to
+ * `users/{userId}/posts`); their refs are accumulated under one entry.
+ */
+export interface DiscoveredCollection {
+    templatePath: string;
+    depth: number;
+    /** First concrete collection path encountered for this template. */
+    examplePath: string;
+    /** All concrete collection refs that share this template path. */
+    refs: CrawlerCollectionRef[];
+    /**
+     * Doc refs accumulated across `refs` during BFS expansion. Sampling
+     * draws from this pool — re-listing would double the listDocuments
+     * cost we already paid during structure discovery.
+     */
+    docRefs: CrawlerDocumentRef[];
+}
+export interface CrawlResult {
+    events: DiscoverEvent[];
+    discovered: Map<string, DiscoveredCollection>;
+    /** Total `listCollections` + `listDocuments` calls — feeds cost reporting. */
+    listOps: number;
+}
+/**
+ * Result of a full crawl (structure + sampling). Augments `CrawlResult`
+ * with the per-templatePath finalized schemas the agent surface consumes.
+ *
+ * `continuation` is present iff the crawl paused at a `maxBatchBytes`
+ * boundary; agents resume by calling `crawl(db, { continuation }, sessions)`.
+ * `complete` is true iff the crawl finished — equivalent to
+ * `continuation === undefined` but more readable at call sites.
+ *
+ * Counter fields (`listOps`, `readOps`) are *cumulative* across batches:
+ * a paused crawl returns the running total so the agent's cost-reporting
+ * doesn't have to do the bookkeeping.
+ */
+export interface FullCrawlResult extends CrawlResult {
+    finalizedSchemas: Map<string, CollectionSchema>;
+    /** `.get()` calls issued during sampling — feeds cost reporting. */
+    readOps: number;
+    /** Opaque resume handle (only present when paused). */
+    continuation?: string;
+    /** True iff the crawl completed (no continuation pending). */
+    complete: boolean;
+    /**
+     * Present iff the crawl was a `dryRun: true` preview. Heuristic
+     * projection of what a full crawl would cost; see {@link CrawlOptions.dryRun}.
+     */
+    dryRunCostEstimate?: DryRunCostEstimate;
+}
+/**
+ * Heuristic cost projection returned by `dryRun: true`. The numbers are
+ * upper-bound estimates — agents should treat them as "no more than"
+ * figures, not exact predictions. Formulas are documented in-line so
+ * consumers can sanity-check.
+ */
+export interface DryRunCostEstimate {
+    /** Number of root collections discovered by the single root listCollections call. */
+    rootCollectionCount: number;
+    /** The root collection IDs (after `rootFilter` is applied, if any). */
+    rootCollectionIds: string[];
+    /** The `maxSamples` value the projection used. */
+    maxSamples: number;
+    /** The `dryRunSubtreeMultiplier` value the projection used. */
+    subtreeMultiplier: number;
+    /**
+     * Projected total `listCollections` + `listDocuments` cost of a real
+     * crawl: `1 + rootCount × subtreeMultiplier`. The `1` is the root
+     * `listCollections`; each subtree contributes one `listDocuments`
+     * call to enumerate docs. Per-doc `listCollections` cost is folded
+     * into the multiplier (a subtree of 3 implies ~2 layers of doc
+     * listings).
+     */
+    estimatedListOps: number;
+    /**
+     * Projected total `.get()` cost of sampling: `rootCount ×
+     * subtreeMultiplier × maxSamples`. Upper bound — `stopOnStable`
+     * early-exit and `cappedByMax` can reduce the actual draw.
+     */
+    estimatedReadOps: number;
+}
+/**
+ * JSON-serializable snapshot of an in-progress crawl. Stored in the
+ * session between batches. Refs (CollectionRef/DocumentRef) carry
+ * methods so they can't be persisted directly — we serialize their
+ * paths and reconstruct via `db.collection(path)` / `db.doc(path)` on
+ * resume.
+ *
+ * Phase invariants:
+ *   - `structure` phase: `frontierPaths` may be non-empty; `samplingQueue`
+ *     is empty.
+ *   - `sampling` phase: `frontierPaths` is empty; `samplingQueue` lists
+ *     the templatePaths still pending. Existing entries in
+ *     `finalizedSchemas` are immutable across the rest of the crawl.
+ */
+export interface PersistedCrawlState {
+    phase: 'structure' | 'sampling';
+    /** Layer index after the last completed structure pass. */
+    currentDepth: number;
+    /** Crawl options carried so resume preserves caps the agent set. */
+    maxDepth: number;
+    /** Concrete collection paths to expand in the next structure layer. */
+    frontierPaths: string[];
+    /** Discovered map serialized — all paths only, refs reconstructed on resume. */
+    discovered: Record<string, PersistedDiscoveredCollection>;
+    /** TemplatePaths still to sample. Drained left-to-right. */
+    samplingQueue: string[];
+    /** Per-templatePath finalized schemas — immutable once set. */
+    finalizedSchemas: Record<string, CollectionSchema>;
+    /** Cumulative cost counters across batches. */
+    listOps: number;
+    readOps: number;
+}
+/** Persisted shape of a `DiscoveredCollection` (refs → paths). */
+export interface PersistedDiscoveredCollection {
+    templatePath: string;
+    depth: number;
+    examplePath: string;
+    refPaths: string[];
+    docRefPaths: string[];
+}
+/**
+ * Map a concrete collection path to its template-path form per Phase 3.1
+ * lock. Doc-id segments become `{singular(parentColl)Id}` so the result
+ * matches Firestore rules' `path.raw` segments under typical naming
+ * conventions (TTT corpus verified: `ttt_lobbies` → `{lobbyId}`).
+ *
+ * Inputs alternate `coll/doc/coll/doc/.../coll`. Length is always odd (a
+ * collection path ends on a collection segment).
+ *
+ * Heuristic — agents needing strict alignment with rules should normalize
+ * both sides before joining. See Risk 6 in the implementation plan.
+ *
+ * Examples:
+ *   `users` → `users`
+ *   `users/uid_1/posts` → `users/{userId}/posts`
+ *   `ttt_lobbies/abc/games/g1/moves` → `ttt_lobbies/{lobbyId}/games/{gameId}/moves`
+ */
+export declare function toTemplatePath(concretePath: string): string;
+/**
+ * Convert a collection ID to the conventional template-variable name a
+ * Firestore rules author would write for its docs. Strips a trailing
+ * snake/dot-cased prefix word so `ttt_lobbies` → `lobbyId` (not
+ * `ttt_lobbieId`).
+ */
+export declare function inferTemplateVariable(collectionId: string): string;
+/**
+ * Walk the Firestore tree breadth-first. Each layer issues its
+ * `listDocuments` + per-doc `listCollections` calls in parallel under a
+ * shared concurrency cap.
+ *
+ * Returns once every reachable collection (within `maxDepth`) is recorded.
+ * The returned `discovered` map is keyed by templatePath; `events` is the
+ * ordered event log emitted during the walk (currently only
+ * `collection_discovered`).
+ */
+export declare function crawlStructure(db: CrawlerFirestore, options?: CrawlOptions): Promise<CrawlResult>;
+/**
+ * Full crawl: discover structure, then sample up to `maxSamples` docs per
+ * discovered templatePath and feed them through the merge layer. Emits
+ * `schema_updated` events for every non-empty merge and `sampling_complete`
+ * once per templatePath.
+ *
+ * **Pause/resume (Item 4.2).** When a `SessionStore` is supplied, the
+ * crawler measures the persisted-state size at two pause boundaries:
+ *
+ *   1. End of every BFS layer in the structure phase
+ *   2. End of every templatePath in the sampling phase
+ *
+ * If the persisted state exceeds `maxBatchBytes` (default 1 MB), the
+ * crawler persists state and returns a `continuation` token. The agent
+ * resumes by passing `{ continuation }` on the next call. Counters
+ * (`listOps`, `readOps`) are cumulative across batches; events are
+ * per-batch only (agents accumulate them themselves).
+ *
+ * Without a `SessionStore`, the crawler runs to completion regardless of
+ * size — single-call mode is unchanged.
+ *
+ * **Continuation lifecycle.** Continuation handles are minted/validated by
+ * the supplied `SessionStore` (see `discover/session.ts`). Malformed,
+ * expired, or evicted tokens surface as a single `error` event and an
+ * otherwise-empty result — agents can re-issue without continuation
+ * per the recovery hint.
+ */
+export declare function crawl(db: CrawlerFirestore, options?: CrawlOptions, sessions?: SessionStore<PersistedCrawlState>): Promise<FullCrawlResult>;
+//# sourceMappingURL=crawler.d.ts.map

package/dist/discover/crawler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawler.d.ts","sourceRoot":"","sources":["../../src/discover/crawler.ts"],"names":[],"mappings":"AA2BA,OAAO,EAAE,YAAY,EAAqB,MAAM,cAAc,CAAC;AAE/D,OAAO,KAAK,EACV,oBAAoB,EACpB,kBAAkB,EAClB,gBAAgB,EACjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EACV,gBAAgB,EAChB,aAAa,EAGd,MAAM,YAAY,CAAC;AAIpB;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,WAAW,YAAY;IAC3B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK,OAAO,CAAC;IAC/C;;;;;OAKG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;;;;OAOG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC;;;;;;;;;;OAUG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;;;;;;OAUG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;;;;;;;;;;;;;OAiBG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;;;;;;;;OAQG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;CAClC;AA4BD;;;;;;;GAOG;AACH,MAAM,WAAW,oBAAoB;IACnC,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,oEAAoE;IACpE,WAAW,EAAE,MAAM,CAAC;IACpB,kEAAkE;IAClE,IAAI,EAAE,oBAAoB,EAAE,CAAC;IAC7B;;;;OAIG;IACH,OAAO,EAAE,kBAAkB,EAAE,CAAC;CAC/B;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,aAAa,EAAE,CAAC;IACxB,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,oBAAoB,CAAC,CAAC;IAC9C,8EAA8E;IAC9E,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,eAAgB,SAAQ,WAAW;IAClD,gBAAgB,EAAE,GAAG,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAChD,oEAAoE;IACpE,OAAO,EAAE,MAAM,CAAC;IAChB,uDAAuD;IACvD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,8DAA8D;IAC9D,QAAQ,EAAE,OAAO,CAAC;IAClB;;;OAGG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CACzC;AAED;;;;;GAKG;AACH,MAAM,WAAW,kBAAkB;IACjC,qFAAqF;IACrF,mBAAmB,EAAE,MAAM,CAAC;IAC5B,uEAAuE;IACvE,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,kDAAkD;IAClD,UAAU,EAAE,MAAM,CAAC;IACnB,+DAA+D;IAC/D,iBAAiB,EAAE,MAAM,CAAC;IAC1B;;;;;;;OAOG;IACH,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;;OAIG;IACH,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAID;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,WAAW,GAAG,UAAU,CAAC;IAChC,2DAA2D;IAC3D,YAAY,EAAE,MAAM,CAAC;IACrB,oEAAoE;IACpE,QAAQ,EAAE,MAAM,CAAC;IACjB,uEAAuE;IACvE,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,gFAAgF;IAChF,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,6BAA6B,CAAC,CAAC;IAC1D,4DAA4D;IAC5D,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,+DAA+D;IAC/D,gBAAgB,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IACnD,+CAA+C;IAC/C,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,kEAAkE;AAClE,MAAM,WAAW,6BAA6B;IAC5C,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAID;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,cAAc,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CAiB3D;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,CAOlE;AAgGD;;;;;;;;;GASG;AACH,wBAAsB,cAAc,CAClC,EAAE,EAAE,gBAAgB,EACpB,OAAO,GAAE,YAAiB,GACzB,OAAO,CAAC,WAAW,CAAC,CA8EtB;AAID;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAsB,KAAK,CACzB,EAAE,EAAE,gBAAgB,EACpB,OAAO,GAAE,YAAiB,EAC1B,QAAQ,CAAC,EAAE,YAAY,CAAC,mBAAmB,CAAC,GAC3C,OAAO,CAAC,eAAe,CAAC,CA8O1B"}