@gscdump/engine 1.4.2 → 1.4.4

package/dist/adapters/node-harness.d.mts

@@ -1,6 +1,6 @@
 import { DataSource, StorageEngine } from "../storage.mjs";
 import { SearchType } from "gscdump/query";
-import { Row, TableName } from "@gscdump/contracts";
+import { Row, TableName, TenantCtx } from "@gscdump/contracts";
 interface NodeHarnessOptions {
   dataDir: string;
   /** Tenant user id. Defaults to `'local'` for single-user CLI installs. */
@@ -18,6 +18,7 @@ interface NodeHarness {
   dataDir: string;
   userId: string;
   siteIdFor: (siteUrl: string) => string;
+  withSitemapMutation: <T>(ctx: TenantCtx, mutate: () => Promise<T>) => Promise<T>;
   runRawSql: (opts: {
     sql: string;
     siteUrl: string;

package/dist/adapters/node-harness.mjs

@@ -11,9 +11,10 @@ function createNodeHarness(opts) {
 	const handle = createNodeDuckDBHandle();
 	const factory = { getDuckDB: async () => handle };
 	const dataSource = createFilesystemDataSource({ rootDir: dataDir });
+	const manifestStore = createFilesystemManifestStore({ path: path.join(dataDir, manifestFilename) });
 	const engine = createStorageEngine({
 		dataSource,
-		manifestStore: createFilesystemManifestStore({ path: path.join(dataDir, manifestFilename) }),
+		manifestStore,
 		codec: createDuckDBCodec(factory),
 		executor: createDuckDBExecutor(factory)
 	});
@@ -41,6 +42,16 @@ function createNodeHarness(opts) {
 		dataDir,
 		userId,
 		siteIdFor: encodeSiteId,
+		withSitemapMutation(ctx, mutate) {
+			if (ctx.userId !== userId) throw new Error(`Sitemap mutation user scope mismatch: expected ${userId}, got ${ctx.userId}`);
+			if (!ctx.siteId) throw new Error("Sitemap mutation requires a site id");
+			return manifestStore.withLock({
+				userId,
+				siteId: ctx.siteId,
+				table: "pages",
+				partition: "__sitemap_entities__"
+			}, mutate);
+		},
 		runRawSql
 	};
 }

package/dist/drizzle-schema.d.mts

@@ -79,7 +79,7 @@ declare const pages: import("drizzle-orm/pg-core/table").PgTableWithColumns<{
       generated: undefined;
     }>;
   };
-  dialect: 'pg';
+  dialect: "pg";
 }>;
 declare const queries: import("drizzle-orm/pg-core/table").PgTableWithColumns<{
   name: "queries";
@@ -161,7 +161,7 @@ declare const queries: import("drizzle-orm/pg-core/table").PgTableWithColumns<{
       generated: undefined;
     }>;
   };
-  dialect: 'pg';
+  dialect: "pg";
 }>;
 declare const countries: import("drizzle-orm/pg-core/table").PgTableWithColumns<{
   name: "countries";
@@ -243,7 +243,7 @@ declare const countries: import("drizzle-orm/pg-core/table").PgTableWithColumns<
       generated: undefined;
     }>;
   };
-  dialect: 'pg';
+  dialect: "pg";
 }>;
 declare const page_queries: import("drizzle-orm/pg-core/table").PgTableWithColumns<{
   name: "page_queries";
@@ -340,7 +340,7 @@ declare const page_queries: import("drizzle-orm/pg-core/table").PgTableWithColum
       generated: undefined;
     }>;
   };
-  dialect: 'pg';
+  dialect: "pg";
 }>;
 /**
  * Per-`(site, search_type, date)` daily totals + device breakdown.
@@ -573,7 +573,7 @@ declare const dates: import("drizzle-orm/pg-core/table").PgTableWithColumns<{
       generated: undefined;
     }>;
   };
-  dialect: 'pg';
+  dialect: "pg";
 }>;
 declare const search_appearance: import("drizzle-orm/pg-core/table").PgTableWithColumns<{
   name: "search_appearance";
@@ -655,7 +655,7 @@ declare const search_appearance: import("drizzle-orm/pg-core/table").PgTableWith
       generated: undefined;
     }>;
   };
-  dialect: 'pg';
+  dialect: "pg";
 }>;
 declare const search_appearance_pages: import("drizzle-orm/pg-core/table").PgTableWithColumns<{
   name: "search_appearance_pages";
@@ -752,7 +752,7 @@ declare const search_appearance_pages: import("drizzle-orm/pg-core/table").PgTab
       generated: undefined;
     }>;
   };
-  dialect: 'pg';
+  dialect: "pg";
 }>;
 declare const search_appearance_queries: import("drizzle-orm/pg-core/table").PgTableWithColumns<{
   name: "search_appearance_queries";
@@ -849,7 +849,7 @@ declare const search_appearance_queries: import("drizzle-orm/pg-core/table").PgT
       generated: undefined;
     }>;
   };
-  dialect: 'pg';
+  dialect: "pg";
 }>;
 declare const search_appearance_page_queries: import("drizzle-orm/pg-core/table").PgTableWithColumns<{
   name: "search_appearance_page_queries";
@@ -961,7 +961,7 @@ declare const search_appearance_page_queries: import("drizzle-orm/pg-core/table"
       generated: undefined;
     }>;
   };
-  dialect: 'pg';
+  dialect: "pg";
 }>;
 declare const hourly_pages: import("drizzle-orm/pg-core/table").PgTableWithColumns<{
   name: "hourly_pages";
@@ -1058,7 +1058,7 @@ declare const hourly_pages: import("drizzle-orm/pg-core/table").PgTableWithColum
       generated: undefined;
     }>;
   };
-  dialect: 'pg';
+  dialect: "pg";
 }>;
 declare const drizzleSchema: {
   pages: import("drizzle-orm/pg-core/table").PgTableWithColumns<{
@@ -1141,7 +1141,7 @@ declare const drizzleSchema: {
         generated: undefined;
       }>;
     };
-    dialect: 'pg';
+    dialect: "pg";
   }>;
   queries: import("drizzle-orm/pg-core/table").PgTableWithColumns<{
     name: "queries";
@@ -1223,7 +1223,7 @@ declare const drizzleSchema: {
         generated: undefined;
       }>;
     };
-    dialect: 'pg';
+    dialect: "pg";
   }>;
   countries: import("drizzle-orm/pg-core/table").PgTableWithColumns<{
     name: "countries";
@@ -1305,7 +1305,7 @@ declare const drizzleSchema: {
         generated: undefined;
       }>;
     };
-    dialect: 'pg';
+    dialect: "pg";
   }>;
   page_queries: import("drizzle-orm/pg-core/table").PgTableWithColumns<{
     name: "page_queries";
@@ -1402,7 +1402,7 @@ declare const drizzleSchema: {
         generated: undefined;
       }>;
     };
-    dialect: 'pg';
+    dialect: "pg";
   }>;
   dates: import("drizzle-orm/pg-core/table").PgTableWithColumns<{
     name: "dates";
@@ -1619,7 +1619,7 @@ declare const drizzleSchema: {
         generated: undefined;
       }>;
     };
-    dialect: 'pg';
+    dialect: "pg";
   }>;
   search_appearance: import("drizzle-orm/pg-core/table").PgTableWithColumns<{
     name: "search_appearance";
@@ -1701,7 +1701,7 @@ declare const drizzleSchema: {
         generated: undefined;
       }>;
     };
-    dialect: 'pg';
+    dialect: "pg";
   }>;
   search_appearance_pages: import("drizzle-orm/pg-core/table").PgTableWithColumns<{
     name: "search_appearance_pages";
@@ -1798,7 +1798,7 @@ declare const drizzleSchema: {
         generated: undefined;
       }>;
     };
-    dialect: 'pg';
+    dialect: "pg";
   }>;
   search_appearance_queries: import("drizzle-orm/pg-core/table").PgTableWithColumns<{
     name: "search_appearance_queries";
@@ -1895,7 +1895,7 @@ declare const drizzleSchema: {
         generated: undefined;
       }>;
     };
-    dialect: 'pg';
+    dialect: "pg";
   }>;
   search_appearance_page_queries: import("drizzle-orm/pg-core/table").PgTableWithColumns<{
     name: "search_appearance_page_queries";
@@ -2007,7 +2007,7 @@ declare const drizzleSchema: {
         generated: undefined;
       }>;
     };
-    dialect: 'pg';
+    dialect: "pg";
   }>;
   hourly_pages: import("drizzle-orm/pg-core/table").PgTableWithColumns<{
     name: "hourly_pages";
@@ -2104,7 +2104,7 @@ declare const drizzleSchema: {
         generated: undefined;
       }>;
     };
-    dialect: 'pg';
+    dialect: "pg";
   }>;
 };
 type DrizzleSchema = typeof drizzleSchema;

package/dist/entities.d.mts

@@ -1,6 +1,7 @@
 import { DataSource } from "./storage.mjs";
 import { ScheduleState } from "./schedule.mjs";
-import { emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPrefix } from "./entity-keys.mjs";
+import { SitemapGenerationKey, emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, parseSitemapUrlsDeltaKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsEventKey, sitemapUrlsEventSeedKey, sitemapUrlsEventsPrefix, sitemapUrlsGenerationKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPendingGenerationKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey, sitemapUrlsReconcileGenerationKey } from "./entity-keys.mjs";
+import { SITEMAP_PROJECTION_GRACE_MS, SitemapProjectionFeed, SitemapProjectionFiles, SitemapProjectionManifest, decodeSitemapProjectionManifest, emptySitemapProjectionManifest, parseSitemapProjectionManifest, selectSitemapProjectionFiles, withSitemapProjectionFeed } from "./sitemap-projection.mjs";
 import { QueryDimDeps, QueryDimMeta, QueryDimRecord, QueryDimStore, buildQueryDimRecords, createQueryDimStore } from "./query-dim.mjs";
 import { TenantCtx } from "@gscdump/contracts";
 /**
@@ -284,9 +285,15 @@ interface SnapshotUrlsResult {
   removed: number;
   kept: number;
   contentHash: string;
-  /** True when contentHash matched prior; the call performed zero writes. */
+  /** True when current membership did not change. Initial history seeding may write. */
   unchanged: boolean;
 }
+interface CompleteSitemapGeneration {
+  _tag: 'complete';
+  id: string;
+  /** Unix epoch milliseconds. */
+  observedAt: number;
+}
 interface ReconcileResult {
   /** Feedpaths that were absent from the live set and had their live URLs pruned. */
   feedpathsPruned: number;
@@ -307,12 +314,12 @@ interface CompactUrlsOptions {
   maxFeedpaths?: number;
 }
 interface CompactUrlsResult {
-  /** Feedpaths whose deltas were folded and deleted by this call. */
+  /** Feedpaths whose active deltas were folded and retired by this call. */
   compactedFeedpaths: number;
   /**
    * Feedpaths that still hold outstanding deltas. Call `compactUrls` again to
-   * continue — no cursor is needed, because a compacted feedpath's deltas are
-   * deleted, so the next call simply doesn't see it.
+   * continue. No cursor is needed because the projection watermark excludes a
+   * compacted feedpath's grace-retained deltas from the next call.
    */
   remainingFeedpaths: number;
 }
@@ -325,6 +332,21 @@ interface DeltaEntry {
   lastmod?: string;
   at: number;
 }
+interface SitemapMembershipEvent {
+  feedpath: string;
+  feedpathHash: string;
+  urlHash: string;
+  op: 'added' | 'removed';
+  loc: string;
+  lastmod?: string;
+  generationId: string;
+  /** Unix epoch milliseconds. */
+  observedAt: number;
+  /** Stable ordering within one generation. */
+  sequence: number;
+  /** Whether this row also mutates the disposable current-state projection. */
+  projectsState: boolean;
+}
 interface DateRange {
   /** YYYY-MM-DD inclusive. */
   from?: string;
@@ -334,27 +356,29 @@ interface DateRange {
 interface LoadUrlsOptions {
   includeRemoved?: boolean;
 }
-interface SitemapStore {
-  /**
-   * Persist a snapshot run. Updates the index + writes one immutable
-   * history doc per record under `history/<feedpathHash>__<capturedAtMs>.json`.
-   */
-  writeSnapshot: (ctx: TenantCtx, records: readonly SitemapRecord[]) => Promise<void>;
+interface SitemapReadStore {
   /** Load the full site index (latest record per feedpath). */
   loadIndex: (ctx: TenantCtx) => Promise<SitemapIndex>;
   /** Fetch the latest snapshot for a feedpath, or undefined. */
   getLatest: (ctx: TenantCtx, path: string) => Promise<SitemapRecord | undefined>;
-  /**
-   * Diff incoming URLs against the prior `urls/index.parquet` partition for
-   * `feedpath`; on change, writes a single delta parquet under
-   * `urls/deltas/YYYY-MM-DD__{feedpathHash}.parquet`. Skipped (0 PUTs) when
-   * `contentHash` matches prior.
-   */
-  snapshotUrls: (ctx: TenantCtx, feedpath: string, urls: readonly ParsedUrl[]) => Promise<SnapshotUrlsResult>;
   /** Stream live (and optionally removed) URL rows for a feedpath. */
   loadUrls: (ctx: TenantCtx, feedpath: string, opts?: LoadUrlsOptions) => AsyncIterable<SitemapUrlRecord>;
   /** Stream all delta entries within `[from, to]` (YYYY-MM-DD inclusive). */
   loadDeltas: (ctx: TenantCtx, dateRange?: DateRange) => AsyncIterable<DeltaEntry>;
+  /** Stream immutable membership events within `[from, to]`. */
+  loadEvents: (ctx: TenantCtx, dateRange?: DateRange) => AsyncIterable<SitemapMembershipEvent>;
+}
+interface SitemapStore extends SitemapReadStore {
+  /**
+   * Persist a snapshot run. Updates the index + writes one immutable
+   * history doc per record under `history/<feedpathHash>__<capturedAtMs>.json`.
+   */
+  writeSnapshot: (ctx: TenantCtx, records: readonly SitemapRecord[]) => Promise<void>;
+  /**
+   * Diff a complete sitemap generation against current state. The immutable
+   * membership event lands before its disposable state delta.
+   */
+  snapshotUrls: (ctx: TenantCtx, generation: CompleteSitemapGeneration, feedpath: string, urls: readonly ParsedUrl[]) => Promise<SnapshotUrlsResult>;
   /**
    * Fold accumulated deltas into the prior index, one feedpath at a time:
    * rewrites each touched feedpath's `by-feed/<hash>/index.parquet` and deletes
@@ -362,9 +386,9 @@ interface SitemapStore {
    * regardless of total site URL count.
    *
    * Optionally bounded in TIME too (`opts.deadlineMs` / `opts.maxFeedpaths`).
-   * A bounded call is safe to stop mid-way: each feedpath's deltas are deleted
-   * as soon as its index is rewritten, so the remainder is simply what the next
-   * call finds. Callers drive this from `remainingFeedpaths`, not a cursor.
+   * A bounded call is safe to stop mid-way: each rewritten feedpath advances
+   * the projection watermark, so the remainder is simply what the next call
+   * finds. Callers drive this from `remainingFeedpaths`, not a cursor.
    */
   compactUrls: (ctx: TenantCtx, opts?: CompactUrlsOptions) => Promise<CompactUrlsResult>;
   /**
@@ -373,22 +397,26 @@ interface SitemapStore {
    * prune URLs *inside* a feedpath that was re-observed; a whole feed dropped
    * from the sitemap list (no `snapshotUrls` call) leaves its URLs frozen-live
    * forever. This is the sidecar mirror of the D1 generation sweep: it rewrites
-   * each dropped feedpath's `by-feed/<hash>/index.parquet` with `removedAt` set
-   * and deletes its outstanding deltas (write-new-base + delete-deltas,
-   * ADR-0002). Bounded per feedpath, so memory stays flat regardless of site
-   * size. Live feedpaths are never touched.
+   * each dropped feedpath's `by-feed/<hash>/index.parquet` with `removedAt` set,
+   * advances its projection watermark, and grace-retires outstanding deltas.
+   * Bounded per feedpath, so memory stays flat regardless of site size. Live
+   * feedpaths are never touched.
    */
-  reconcile: (ctx: TenantCtx, opts: {
+  reconcile: (ctx: TenantCtx, generation: CompleteSitemapGeneration, opts: {
     liveFeedpaths: readonly string[];
-    at?: number;
   }) => Promise<ReconcileResult>;
 }
-interface CreateSitemapStoreOptions {
+interface CreateSitemapReadStoreOptions {
   dataSource: DataSource;
   /** Override the feedpath hash (test seam). */
   hash?: (path: string) => string;
+}
+type SitemapMutation = <T>(ctx: TenantCtx, fn: () => Promise<T>) => Promise<T>;
+interface CreateSitemapStoreOptions extends CreateSitemapReadStoreOptions {
+  withMutation: SitemapMutation;
   now?: () => number;
 }
+declare function createSitemapReadStore(opts: CreateSitemapReadStoreOptions): SitemapReadStore;
 declare function createSitemapStore(opts: CreateSitemapStoreOptions): SitemapStore;
 interface IndexingMetadataRecord {
   url: string;
@@ -432,4 +460,4 @@ interface CreateEmptyTypesStoreOptions {
   now?: () => number;
 }
 declare function createEmptyTypesStore(opts: CreateEmptyTypesStoreOptions): EmptyTypesStore;
-export { CompactUrlsOptions, CompactUrlsResult, CreateEmptyTypesStoreOptions, CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, CreateSitemapStoreOptions, DateRange, DeltaEntry, EmptyTypesDoc, EmptyTypesStore, INSPECTION_HISTORY_MAX_BYTES, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, InspectionEventRow, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, LoadUrlsOptions, ParsedUrl, QueryDimDeps, QueryDimMeta, QueryDimRecord, QueryDimStore, ReconcileResult, SitemapHistoryDoc, SitemapIndex, SitemapRecord, SitemapStore, SitemapUrlRecord, SnapshotUrlsResult, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapStore, emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPrefix };
+export { CompactUrlsOptions, CompactUrlsResult, CompleteSitemapGeneration, CreateEmptyTypesStoreOptions, CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, CreateSitemapReadStoreOptions, CreateSitemapStoreOptions, DateRange, DeltaEntry, EmptyTypesDoc, EmptyTypesStore, INSPECTION_HISTORY_MAX_BYTES, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, InspectionEventRow, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, LoadUrlsOptions, ParsedUrl, QueryDimDeps, QueryDimMeta, QueryDimRecord, QueryDimStore, ReconcileResult, SITEMAP_PROJECTION_GRACE_MS, SitemapGenerationKey, SitemapHistoryDoc, SitemapIndex, SitemapMembershipEvent, SitemapMutation, SitemapProjectionFeed, SitemapProjectionFiles, SitemapProjectionManifest, SitemapReadStore, SitemapRecord, SitemapStore, SitemapUrlRecord, SnapshotUrlsResult, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapReadStore, createSitemapStore, decodeSitemapProjectionManifest, emptySitemapProjectionManifest, emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, parseSitemapProjectionManifest, parseSitemapUrlsDeltaKey, queryDimMetaKey, queryDimParquetKey, selectSitemapProjectionFiles, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsEventKey, sitemapUrlsEventSeedKey, sitemapUrlsEventsPrefix, sitemapUrlsGenerationKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPendingGenerationKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey, sitemapUrlsReconcileGenerationKey, withSitemapProjectionFeed };