@gscdump/engine 1.3.2 → 1.4.1

package/dist/index.d.mts

@@ -1,191 +1,15 @@
-import { CodecCtx, CompactionThresholds, CompactionTier, DataSource, EngineOptions, FileSetRef, GcCtx, Grain, ListLiveFilter, LockScope, ManifestEntry, ManifestPurgeResult, ManifestStore, ParquetCodec, PurgeFilter, PurgeResult, PurgeUrlsResult, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryProfiler, QueryResult, QuerySpan, Row, RunSQLOptions, SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, TableName, TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult } from "./_chunks/storage.mjs";
-import { DuckDBFactory, DuckDBHandle, SnapshotIndex, createDuckDBCodec, createDuckDBExecutor } from "./_chunks/snapshot.mjs";
-import { EngineError } from "./_chunks/errors.mjs";
-import { InspectionVerdict, SchedulePolicy, ScheduleState, fixedPolicy, inspectionPolicy, sitemapPolicy } from "./_chunks/schedule.mjs";
-import { IcebergTableName, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult } from "./_chunks/sink.mjs";
-import { Result } from "gscdump/result";
-import { SearchType as SearchType$1 } from "gscdump/query";
-import { Row as Row$1, TableName as TableName$1, TenantCtx as TenantCtx$1 } from "@gscdump/contracts";
-interface R2ObjectMetadata {
-  etag: string;
-}
-interface R2ObjectBody extends R2ObjectMetadata {
-  text: () => Promise<string>;
-}
-interface R2ListResult {
-  objects: Array<{
-    key: string;
-  }>;
-  truncated: boolean;
-  cursor?: string;
-}
-interface R2ConditionalPutOptions {
-  /**
-   * Workers-binding-style precondition. `etagMatches` rejects with `null`
-   *  return on mismatch; `etagDoesNotMatch: '*'` rejects if the key exists.
-   */
-  onlyIf?: {
-    etagMatches?: string;
-    etagDoesNotMatch?: string;
-  };
-}
-/**
- * Minimal Cloudflare R2 binding shape needed for the manifest CAS loop.
- * Structurally compatible with Cloudflare's `R2Bucket` Workers API.
- */
-interface R2ManifestBucketLike {
-  get: (key: string) => Promise<R2ObjectBody | null>;
-  put: (key: string, bytes: string | Uint8Array, options?: R2ConditionalPutOptions) => Promise<R2ObjectMetadata | null>;
-  list: (options?: {
-    prefix?: string;
-    cursor?: string;
-    limit?: number;
-  }) => Promise<R2ListResult>;
-  /**
-   * Bulk delete. Required by {@link ManifestStore.purgeTenant}. Cloudflare's
-   * `R2Bucket.delete` accepts a single key or a string[] batch; both shapes
-   * work here.
-   */
-  delete: (keys: string | string[]) => Promise<void>;
-}
-/**
- * CAS lifecycle events emitted by the manifest store. Consumers wire these
- * into metrics (prom-client, console.table, the contention harness) to
- * measure rejection rate and latency under real R2 load.
- */
-type R2ManifestEvent = {
-  kind: 'cas-attempt';
-  siteId: string;
-  table: TableName;
-  attempt: number;
-} | {
-  kind: 'cas-rejected';
-  siteId: string;
-  table: TableName;
-  attempt: number;
-} | {
-  kind: 'cas-committed';
-  siteId: string;
-  table: TableName;
-  attempts: number;
-};
-interface CreateR2ManifestStoreOptions {
-  bucket: R2ManifestBucketLike;
-  /** Tenant scope. All shard keys are prefixed `u_<userId>/manifest/...`. */
-  userId: string;
-  /** Override the snapshot version-id generator. Defaults to `${ts}-${random}`. */
-  newSnapshotId?: () => string;
-  now?: () => number;
-  /** Maximum CAS retries before giving up. Defaults to 8. */
-  maxRetries?: number;
-  /**
-   * Optional telemetry hook. Fired synchronously from the CAS loop on each
-   * attempt, rejection, and successful commit. Must not throw; exceptions
-   * propagate and will fail the mutation.
-   */
-  onEvent?: (event: R2ManifestEvent) => void;
-}
-declare function createR2ManifestStore(opts: CreateR2ManifestStoreOptions): ManifestStore;
-declare function coerceRow(row: Row$1): Row$1;
-declare function coerceRows(rows: readonly Row$1[]): Row$1[];
-declare const MAX_DAY_BYTES: number;
-declare function createStorageEngine(opts: EngineOptions): StorageEngine;
-declare function dayPartition(date: string): string;
-/**
- * Hourly partition keyed by the PT calendar day (`YYYY-MM-DD`). One parquet
- * per day holds 24 hourly buckets — read-merge-write keeps `(url, hour)`
- * idempotency across retries. Names sort lexically alongside daily ones but
- * never collide because of the `hourly/` prefix.
- */
-declare function hourPartition(date: string): string;
-/**
- * Default `searchType` for entries written before the field landed and for
- * sync paths that don't request a specific type. GSC's own default; the
- * vast majority of stored data is web-search.
- */
-declare const DEFAULT_SEARCH_TYPE: SearchType$1;
-declare function objectKey(ctx: TenantCtx$1, table: TableName$1, partition: string, version: number, searchType?: SearchType$1): string;
-/**
- * Resolve the search type for an entry, defaulting legacy entries to `web`.
- * Use this anywhere code needs to bucket entries by searchType.
- */
-declare function inferSearchType(entry: Pick<ManifestEntry, 'searchType'>): SearchType$1;
-/**
- * Infer the tier for an entry that pre-dates the `tier` field. Daily files
- * are `raw`; monthly files are `d30`. Anything else (already migrated, or
- * a partition shape we haven't seen) returns undefined and the caller must
- * decide how to handle it.
- */
-declare function inferLegacyTier(entry: Pick<ManifestEntry, 'partition' | 'tier'>): CompactionTier | undefined;
-/**
- * Build a {@link QueryProfiler} that records each closed span to `sink`.
- *
- * `start(name, meta)` stamps the open time and returns an `end` thunk; calling
- * `end(extra)` records `{ name, ms, meta }` with `extra` merged over the
- * open-time `meta` (so completion-only facts — row counts, buffered-file
- * counts — land on the same span). `now` is injectable for deterministic
- * tests; it defaults to `Date.now`.
- */
-declare function createQueryProfiler(sink: (span: QuerySpan) => void, now?: () => number): QueryProfiler;
-/**
- * A profiler that accumulates closed spans into an array — for tests, the CLI
- * `query` command, or any ad-hoc "where did the time go" probe. The returned
- * `spans` array is filled as spans close, in completion order.
- */
-declare function collectSpans(now?: () => number): {
-  profiler: QueryProfiler;
-  spans: QuerySpan[];
-};
-/** A row as stored by the fake — data columns plus the injected identity columns. */
-type StoredRow = Row & {
-  site_id: string;
-  search_type: string;
-};
-interface InMemorySink extends Sink {
-  /** All rows accepted, in partition order. */
-  readonly rows: readonly StoredRow[];
-  /** Rows for one table. */
-  rowsFor: (table: IcebergTableName) => StoredRow[];
-  /** Rows for one exact slice partition. */
-  rowsForSlice: (slice: SinkSlice) => StoredRow[];
-  /** `true` once `close()` has run. */
-  readonly closed: boolean;
-  /** Drop everything — handy between test cases. */
-  reset: () => void;
-}
-/**
- * Create an in-memory append-only `Sink`.
- */
-declare function createInMemorySink(): InMemorySink;
-type SyncTableName = Extract<TableName$1, 'pages' | 'queries' | 'countries' | 'page_queries' | 'dates'>;
-declare const TABLES_BY_SEARCH_TYPE: Record<SearchType, readonly SyncTableName[]>;
-declare function parseEnabledSearchTypes(raw: string | null | undefined): SearchType[];
-/**
- * Errors-as-values core for {@link validateEnabledSearchTypes}: returns a typed
- * `invalid-search-types` `EngineError` instead of throwing, so a host saving the
- * persisted config can map a bad value to a 4xx rather than a 500.
- */
-declare function validateEnabledSearchTypesResult(value: unknown): Result<SearchType[], EngineError>;
-declare function validateEnabledSearchTypes(value: unknown): SearchType[];
-declare const TABLE_TIERS: {
-  readonly pages: "critical";
-  readonly queries: "critical";
-  readonly countries: "standard";
-  readonly dates: "standard";
-  readonly page_queries: "extended";
-};
-type TieredTableName = keyof typeof TABLE_TIERS;
-type TableTier = 'critical' | 'standard' | 'extended';
-type DateWeight = 'fresh' | 'recent' | 'historical';
-declare function getTableTier(table: string): TableTier;
-declare function getTablesForTier(tier: TableTier): TieredTableName[];
-declare function getDateWeight(date: string, now?: Date): DateWeight;
-declare const TIER_PRIORITY: Record<TableTier, number>;
-declare const WEIGHT_PRIORITY: Record<DateWeight, number>;
-declare const MAX_GSC_PAGES_R2 = 40;
-declare const ROW_LIMIT_R2 = 25000;
-declare const MIN_SYNC_IMPRESSIONS = 1;
-declare const MIN_COUNTRY_IMPRESSIONS = 10;
-declare const MAX_SITEMAP_URLS_PER_SITE = 50000;
-declare const MAX_TRACKED_URLS_PER_SITE = 200000;
+import { CompactionThresholds } from "./compaction.mjs";
+import { CodecCtx, CompactionTier, DataSource, EngineOptions, FileSetRef, GcCtx, Grain, ListLiveFilter, LockScope, ManifestEntry, ManifestPurgeResult, ManifestStore, ParquetCodec, PurgeFilter, PurgeResult, PurgeUrlsResult, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryProfiler, QueryResult, QuerySpan, Row, RunSQLOptions, SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, TableName, TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult } from "./storage.mjs";
+import { DuckDBFactory, DuckDBHandle, createDuckDBCodec, createDuckDBExecutor } from "./duckdb.mjs";
+import { SnapshotIndex } from "./snapshot.mjs";
+import { InspectionVerdict, SchedulePolicy, ScheduleState, fixedPolicy, inspectionPolicy, sitemapPolicy } from "./schedule.mjs";
+import { Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult } from "./sink.mjs";
+import { CreateR2ManifestStoreOptions, R2ManifestBucketLike, R2ManifestEvent, createR2ManifestStore } from "./adapters/r2-manifest.mjs";
+import { coerceRow, coerceRows } from "./coerce.mjs";
+import { MAX_DAY_BYTES, createStorageEngine } from "./engine.mjs";
+import { DEFAULT_SEARCH_TYPE, dayPartition, hourPartition, inferLegacyTier, inferSearchType, objectKey } from "./layout.mjs";
+import { collectSpans, createQueryProfiler } from "./profile.mjs";
+import { InMemorySink, StoredRow, createInMemorySink } from "./sinks/in-memory-sink.mjs";
+import "./sinks/index.mjs";
+import { DateWeight, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, ROW_LIMIT_R2, SyncTableName, TABLES_BY_SEARCH_TYPE, TABLE_TIERS, TIER_PRIORITY, TableTier, TieredTableName, WEIGHT_PRIORITY, getDateWeight, getTableTier, getTablesForTier, parseEnabledSearchTypes, validateEnabledSearchTypes, validateEnabledSearchTypesResult } from "./sync-config.mjs";
 export { type CodecCtx, type CompactionThresholds, type CompactionTier, type CreateR2ManifestStoreOptions, DEFAULT_SEARCH_TYPE, type DataSource, type DateWeight, type DuckDBFactory, type DuckDBHandle, type EngineOptions, type FileSetRef, type GcCtx, type Grain, type InMemorySink, type InspectionVerdict, type ListLiveFilter, type LockScope, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryProfiler, type QueryResult, type QuerySpan, type R2ManifestBucketLike, type R2ManifestEvent, ROW_LIMIT_R2, type Row, type RunSQLOptions, type SchedulePolicy, type ScheduleState, type SearchType, type Sink, type SinkCapabilities, type SinkCloseResult, type SinkOptions, type SinkSlice, type SinkWriteResult, type SnapshotIndex, type StorageEngine, type StoredRow, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, type SyncTableName, TABLES_BY_SEARCH_TYPE, TABLE_TIERS, TIER_PRIORITY, type TableName, type TableTier, type TenantCtx, type TieredTableName, WEIGHT_PRIORITY, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, coerceRow, coerceRows, collectSpans, createDuckDBCodec, createDuckDBExecutor, createInMemorySink, createQueryProfiler, createR2ManifestStore, createStorageEngine, dayPartition, fixedPolicy, getDateWeight, getTableTier, getTablesForTier, hourPartition, inferLegacyTier, inferSearchType, inspectionPolicy, objectKey, parseEnabledSearchTypes, sitemapPolicy, validateEnabledSearchTypes, validateEnabledSearchTypesResult };