@gscdump/engine 0.32.6 → 0.32.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -303,7 +303,8 @@ declare function icebergAppend({
303
303
  tableUrl,
304
304
  resolver,
305
305
  records,
306
- sortOrderId
306
+ sortOrderId,
307
+ snapshotProperties
307
308
  }: {
308
309
  catalog: Catalog;
309
310
  namespace?: string | string[] | undefined;
@@ -312,6 +313,7 @@ declare function icebergAppend({
312
313
  resolver?: Resolver | undefined;
313
314
  records: Record<string, any>[];
314
315
  sortOrderId?: number | undefined;
316
+ snapshotProperties?: Record<string, string> | undefined;
315
317
  }): Promise<TableMetadata>;
316
318
  /**
317
319
  * Create a new table. REST: delegates to the catalog's create endpoint.
@@ -3216,7 +3216,7 @@ async function prepareAppend({ tableUrl, metadata, records, resolver, sortOrderI
3216
3216
  writtenFiles: [...writtenDataFiles.map((f) => f.path), manifestPath]
3217
3217
  };
3218
3218
  }
3219
- async function stageSnapshotForAppend({ tableUrl, metadata, prepared, resolver }) {
3219
+ async function stageSnapshotForAppend({ tableUrl, metadata, prepared, resolver, snapshotProperties }) {
3220
3220
  if (!tableUrl) throw new Error("tableUrl is required");
3221
3221
  if (!resolver?.writer) throw new Error("resolver.writer is required");
3222
3222
  const sequenceNumber = BigInt(metadata["last-sequence-number"] ?? 0) + 1n;
@@ -3254,7 +3254,8 @@ async function stageSnapshotForAppend({ tableUrl, metadata, prepared, resolver }
3254
3254
  "total-data-files": String(prevTotals.files + BigInt(prepared.addedDataFilesCount)),
3255
3255
  "total-delete-files": "0",
3256
3256
  "total-position-deletes": "0",
3257
- "total-equality-deletes": "0"
3257
+ "total-equality-deletes": "0",
3258
+ ...snapshotProperties ?? {}
3258
3259
  };
3259
3260
  return await buildSnapshotUpdate({
3260
3261
  tableUrl,
@@ -3370,7 +3371,7 @@ const DEFAULT_RETRY = Object.freeze({
3370
3371
  factor: 2,
3371
3372
  totalTimeoutMs: 1800 * 1e3
3372
3373
  });
3373
- async function icebergAppend({ catalog, namespace, table, tableUrl, resolver, records, sortOrderId }) {
3374
+ async function icebergAppend({ catalog, namespace, table, tableUrl, resolver, records, sortOrderId, snapshotProperties }) {
3374
3375
  const ctx = await loadTable({
3375
3376
  catalog,
3376
3377
  namespace,
@@ -3396,7 +3397,8 @@ async function icebergAppend({ catalog, namespace, table, tableUrl, resolver, re
3396
3397
  tableUrl: workingCtx.tableUrl,
3397
3398
  metadata: workingCtx.metadata,
3398
3399
  prepared,
3399
- resolver: requireResolver(workingCtx.resolver, "icebergAppend")
3400
+ resolver: requireResolver(workingCtx.resolver, "icebergAppend"),
3401
+ snapshotProperties
3400
3402
  })
3401
3403
  });
3402
3404
  }
@@ -283,6 +283,14 @@ interface CommitRetryOptions {
283
283
  sleep?: (ms: number) => Promise<void>;
284
284
  /** Injectable RNG for the jitter — tests pass a deterministic value. */
285
285
  random?: () => number;
286
+ /**
287
+ * Idempotency token stamped into the appended snapshot's summary
288
+ * (`gscdump.append-id`) and matched by the landed-check before any outer
289
+ * retry. Generated per call (`crypto.randomUUID`) so it is STABLE across this
290
+ * call's retries but UNIQUE per call — a content hash would risk a false
291
+ * "already landed" skip (silent data loss). Injectable for deterministic tests.
292
+ */
293
+ appendId?: string;
286
294
  }
287
295
  /**
288
296
  * True when `err` is an R2 Data Catalog commit rate-limit response
@@ -297,15 +305,18 @@ declare function isCommitRateLimited(err: unknown): boolean;
297
305
  * retries 412/409 internally; 429 is the gap this closes. Non-429 errors
298
306
  * (and 429s that survive every attempt) propagate unchanged.
299
307
  *
300
- * RESIDUAL RISK re-upload orphans. icebird's `icebergAppend` prepares the
301
- * data + manifest files ONCE, outside its internal 412/409 retry loop, so
302
- * those retries never re-upload data. A 429 that escapes that loop and is
303
- * retried HERE re-runs the whole `icebergAppend` call, which re-prepares and
304
- * re-uploads the data files; the previous attempt's parquet objects become
305
- * orphans (referenced by no snapshot). 429s should be rare and clear within
306
- * an attempt or two, so orphan volume is small, and R2 orphan-file cleanup
307
- * reclaims them. Eliminating the 429 source entirely (per-table commit
308
- * coalescing) is assessed in the Phase-1.5 report.
308
+ * IDEMPOTENT RE-RUN. A 429 that escapes icebird's internal loop is retried HERE
309
+ * by re-running the whole `icebergAppend`, which re-prepares + re-uploads the
310
+ * data files. That is safe ONLY if the prior attempt's commit did NOT land — but
311
+ * R2 Data Catalog can apply a commit and STILL return 429 (rate-limit on the
312
+ * response path), and a lost ack looks identical. A blind re-run then appends a
313
+ * SECOND copy of the same rows silent double-count (observed: a re-resynced
314
+ * site read its true totals). To close this, every attempt stamps a per-call
315
+ * `gscdump.append-id` into the snapshot summary, and BEFORE retrying or giving
316
+ * up we reload the table and check whether that id already landed
317
+ * ({@link appendAlreadyLanded}); if so the append succeeded and we return without
318
+ * re-appending. A non-landed 429 still re-runs (the previous attempt's parquet is
319
+ * an orphan, reclaimed by R2 cleanup) — same as before, no double.
309
320
  */
310
321
  declare function icebergAppendRetrying(args: Parameters<typeof icebergAppend>[0], options?: CommitRetryOptions): Promise<void>;
311
322
  /**
@@ -443,6 +454,10 @@ interface SinkCapabilities {
443
454
  /** Whether the sink exposes partition overwrite semantics. */
444
455
  canOverwrite?: boolean;
445
456
  }
457
+ /** Partition-overwrite writer for revision paths. */
458
+ interface SliceOverwriteWriter {
459
+ overwriteSlice: (slice: SinkSlice, rows: readonly Row$1[]) => Promise<SinkWriteResult>;
460
+ }
446
461
  /**
447
462
  * Outcome of `Sink.close()` — which tables' buffered rows reached durable
448
463
  * storage and which failed.
@@ -528,4 +543,4 @@ interface LocalIcebergSinkOptions extends SinkOptions {
528
543
  /** S3-compatible warehouse location (POC: MinIO). */
529
544
  warehouse: string;
530
545
  }
531
- export { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, LocalIcebergSinkOptions, PartitionKeyEncoding, SEARCH_TYPE_INT, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };
546
+ export { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, LocalIcebergSinkOptions, PartitionKeyEncoding, SEARCH_TYPE_INT, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, SliceOverwriteWriter, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };
@@ -173,6 +173,8 @@ async function connectIcebergCatalog(config, opts = {}) {
173
173
  namespace: config.namespace
174
174
  };
175
175
  }
176
+ const APPEND_ID_SUMMARY_KEY = "gscdump.append-id";
177
+ const APPEND_LANDED_SCAN_DEPTH = 25;
176
178
  function isCommitRateLimited(err) {
177
179
  if (err && typeof err === "object" && err.status === 429) return true;
178
180
  const msg = (err instanceof Error ? err.message : String(err)).toLowerCase();
@@ -187,14 +189,35 @@ async function icebergAppendRetrying(args, options = {}) {
187
189
  const maxDelayMs = options.maxDelayMs ?? 2e4;
188
190
  const sleep = options.sleep ?? defaultCommitSleep;
189
191
  const random = options.random ?? Math.random;
192
+ const appendId = options.appendId ?? globalThis.crypto.randomUUID();
193
+ const stampedArgs = {
194
+ ...args,
195
+ snapshotProperties: {
196
+ ...args.snapshotProperties,
197
+ [APPEND_ID_SUMMARY_KEY]: appendId
198
+ }
199
+ };
190
200
  for (let attempt = 0; attempt < maxAttempts; attempt++) {
191
- const err = await icebergAppend(args).then(() => void 0, (e) => e);
201
+ const err = await icebergAppend(stampedArgs).then(() => void 0, (e) => e);
192
202
  if (err === void 0) return;
203
+ if (await appendAlreadyLanded(args, appendId).catch(() => false)) return;
193
204
  if (!isCommitRateLimited(err) || attempt === maxAttempts - 1) throw err;
194
205
  const ceiling = Math.min(maxDelayMs, baseDelayMs * 2 ** attempt);
195
206
  await sleep(Math.floor(random() * ceiling));
196
207
  }
197
208
  }
209
+ async function appendAlreadyLanded(args, appendId) {
210
+ const a = args;
211
+ if (a.catalog?.type !== "rest" || a.namespace == null || a.table == null) return false;
212
+ const { metadata } = await restCatalogLoadTable(a.catalog, {
213
+ namespace: a.namespace,
214
+ table: a.table
215
+ });
216
+ const snapshots = metadata.snapshots ?? [];
217
+ const from = Math.max(0, snapshots.length - APPEND_LANDED_SCAN_DEPTH);
218
+ for (let i = snapshots.length - 1; i >= from; i--) if (snapshots[i]?.summary?.[APPEND_ID_SUMMARY_KEY] === appendId) return true;
219
+ return false;
220
+ }
198
221
  async function ensureIcebergNamespace(conn) {
199
222
  await restCatalogCreateNamespace(conn.catalog, { namespace: conn.namespace }).catch(() => {});
200
223
  }
@@ -1,4 +1,5 @@
1
- import { IcebergS3Config, LocalIcebergSinkOptions, Sink } from "./_chunks/sink.mjs";
1
+ import { Row } from "./_chunks/storage.mjs";
2
+ import { IcebergS3Config, LocalIcebergSinkOptions, Sink, SinkCloseResult, SliceOverwriteWriter } from "./_chunks/sink.mjs";
2
3
  /** Full `LocalIcebergSink` options — extends the frozen contract options. */
3
4
  interface LocalIcebergSinkFullOptions extends LocalIcebergSinkOptions {
4
5
  /** S3 credentials for the warehouse. Defaults to the POC MinIO creds. */
@@ -21,4 +22,162 @@ interface LocalIcebergSink extends Sink {
21
22
  * use this sink must skip when the stack is unreachable.
22
23
  */
23
24
  declare function createLocalIcebergSink(options: LocalIcebergSinkFullOptions): LocalIcebergSink;
24
- export { type LocalIcebergSink, type LocalIcebergSinkFullOptions, type LocalIcebergSinkOptions, createLocalIcebergSink };
25
+ /** Connection details for the Iceberg REST catalog the writer targets. */
26
+ interface OverwriteWriterCatalogConfig {
27
+ /**
28
+ * Iceberg REST catalog endpoint, e.g. `http://localhost:8181` (POC) or the
29
+ * R2 Data Catalog endpoint (prod).
30
+ */
31
+ catalogUri: string;
32
+ /** Catalog namespace the 5 fact tables live under, e.g. `gsc`. */
33
+ namespace: string;
34
+ /** Warehouse identifier the catalog resolves table locations under. */
35
+ warehouse: string;
36
+ /** S3-compatible object store backing the catalog. */
37
+ s3: IcebergS3Config;
38
+ /** Bearer token for a token-authed REST catalog (R2 Data Catalog). */
39
+ catalogToken?: string;
40
+ }
41
+ /**
42
+ * The JSON job wire-format shared with `scripts/iceberg-writer.py` — identical
43
+ * to the shape `LocalIcebergSink` already emits, so one PyIceberg script backs
44
+ * both the local sink and this writer. `op` is always `overwrite` here.
45
+ */
46
+ interface OverwriteJob {
47
+ op: 'overwrite';
48
+ catalogUri: string;
49
+ namespace: string;
50
+ warehouse: string;
51
+ s3: IcebergS3Config;
52
+ table: string;
53
+ spec: unknown;
54
+ siteId: string;
55
+ searchType: string;
56
+ date: string;
57
+ /** Data columns only — the script injects `site_id` / `search_type`. */
58
+ rows: readonly Row[];
59
+ }
60
+ /**
61
+ * Per-site delete job — removes EVERY row for one `siteId` across a whole
62
+ * table (all dates / search types). Recovery op for clearing a corrupt site
63
+ * (e.g. a cross-run append double) from a shared per-team shard before a clean
64
+ * re-backfill, without touching sibling sites. Carries no rows. `siteId` is a
65
+ * `number` for an INT-encoded shard, a `string` for STRING encoding.
66
+ */
67
+ interface DeleteJob {
68
+ op: 'delete';
69
+ catalogUri: string;
70
+ namespace: string;
71
+ warehouse: string;
72
+ s3: IcebergS3Config;
73
+ /** Bearer token for a token-authed REST catalog (R2 Data Catalog). */
74
+ catalogToken?: string;
75
+ table: string;
76
+ siteId: string | number;
77
+ }
78
+ /** Any job the shared PyIceberg backend can execute. */
79
+ type PyIcebergJob = OverwriteJob | DeleteJob;
80
+ /** Result wire-format from the PyIceberg backend. */
81
+ interface OverwriteJobResult {
82
+ /** Rows written (emit/overwrite) or removed (delete). */
83
+ rowCount?: number;
84
+ error?: string;
85
+ }
86
+ /**
87
+ * A transport that executes one PyIceberg job and returns its result.
88
+ * `subprocessBackend` and `httpBackend` are the two shipped implementations;
89
+ * tests inject a fake. The param is the `OverwriteJob | DeleteJob` union, so a
90
+ * backend value is still assignable where the narrower `OverwriteBackend` is
91
+ * expected (a wider-param function accepts the narrower call).
92
+ */
93
+ type OverwriteBackend = (job: PyIcebergJob) => Promise<OverwriteJobResult>;
94
+ interface IcebergOverwriteWriterOptions {
95
+ catalog: OverwriteWriterCatalogConfig;
96
+ /** The transport that runs the PyIceberg job. */
97
+ backend: OverwriteBackend;
98
+ }
99
+ /**
100
+ * The trailing-window Iceberg overwrite writer. Conforms to the frozen
101
+ * `SliceOverwriteWriter` contract; `overwriteSlice` builds one PyIceberg
102
+ * overwrite job per call and runs it through the configured backend.
103
+ */
104
+ interface IcebergOverwriteWriter extends SliceOverwriteWriter {
105
+ /** Release any backend resources. Idempotent. */
106
+ close: () => Promise<SinkCloseResult>;
107
+ }
108
+ /**
109
+ * Build a trailing-window Iceberg overwrite writer over the given backend.
110
+ * Pure — all I/O is the backend's; the writer only shapes the job.
111
+ */
112
+ declare function createIcebergOverwriteWriter(opts: IcebergOverwriteWriterOptions): IcebergOverwriteWriter;
113
+ /** Options for the PyIceberg subprocess backend. */
114
+ interface SubprocessBackendOptions {
115
+ /** Python interpreter. Defaults to `$GSCDUMP_ICEBERG_PYTHON` then `python3`. */
116
+ python?: string;
117
+ /**
118
+ * Path to the PyIceberg writer script. Defaults to
119
+ * `<engine>/scripts/iceberg-writer.py`.
120
+ */
121
+ writerScript?: string;
122
+ }
123
+ /**
124
+ * Backend that runs the PyIceberg overwrite by spawning the shared writer
125
+ * script (`scripts/iceberg-writer.py`). Node-only — used by local integration
126
+ * tests and a Node-side scheduled job box. The script reads the job JSON on
127
+ * stdin and writes `{rowCount}` / `{error}` on stdout.
128
+ */
129
+ declare function subprocessBackend(opts?: SubprocessBackendOptions): OverwriteBackend;
130
+ /** Options for the PyIceberg HTTP backend (the Cloudflare Container in prod). */
131
+ interface HttpBackendOptions {
132
+ /**
133
+ * URL of the PyIceberg overwrite service — a POST endpoint accepting the
134
+ * `OverwriteJob` JSON and returning `OverwriteJobResult`.
135
+ */
136
+ endpoint: string;
137
+ /** Optional bearer token for the service. */
138
+ token?: string;
139
+ /** Injectable fetch — defaults to the global. */
140
+ fetch?: typeof fetch;
141
+ }
142
+ /**
143
+ * Backend that POSTs the overwrite job to a PyIceberg HTTP service — the
144
+ * Cloudflare Container running `iceberg-writer.py` behind an HTTP shim. This
145
+ * is the prod transport: the request-path Worker holds no Python and never
146
+ * does an Iceberg metadata commit itself.
147
+ */
148
+ declare function httpBackend(opts: HttpBackendOptions): OverwriteBackend;
149
+ /**
150
+ * Adapt an `IcebergOverwriteWriter` into a full `Sink` whose `emit` is also a
151
+ * partition-overwrite — convenient for tests so a single Iceberg writer backs
152
+ * both append and revision paths against the local stack. Prod keeps `emit`
153
+ * on the Pipeline; only `overwriteSlice` delegates here.
154
+ */
155
+ declare function overwriteWriterAsSink(writer: IcebergOverwriteWriter): Sink & SliceOverwriteWriter;
156
+ /** Per-table outcome of a {@link deleteSiteFromShard} run. */
157
+ interface DeleteSiteResult {
158
+ table: string;
159
+ /** Rows removed, when the backend reported a count. */
160
+ rowCount?: number;
161
+ /** Present when this table's delete failed; the sweep continues regardless. */
162
+ error?: string;
163
+ }
164
+ /**
165
+ * Remove every row for one `siteId` from a per-team Iceberg shard, across the
166
+ * given tables. Recovery op: clear a corrupt site (e.g. a cross-run append
167
+ * double) before a clean re-backfill, leaving sibling sites in the shared shard
168
+ * untouched. `site_id` is an identity partition column, so PyIceberg prunes
169
+ * whole data files (no row-level delete files).
170
+ *
171
+ * Runs SEQUENTIALLY (one commit per table at a time) so the per-table catalog
172
+ * commit-rate ceiling is never tripped. A table failure is captured in its
173
+ * result row, not thrown — the sweep always attempts every table so a partial
174
+ * failure is visible and re-runnable (delete is idempotent: re-deleting an
175
+ * already-empty site is a no-op).
176
+ */
177
+ declare function deleteSiteFromShard(args: {
178
+ catalog: OverwriteWriterCatalogConfig;
179
+ backend: OverwriteBackend;
180
+ siteId: string | number;
181
+ tables: readonly string[];
182
+ }): Promise<DeleteSiteResult[]>;
183
+ export { type DeleteJob, type DeleteSiteResult, type IcebergOverwriteWriter, type IcebergOverwriteWriterOptions, type LocalIcebergSink, type LocalIcebergSinkFullOptions, type LocalIcebergSinkOptions, type OverwriteBackend, type OverwriteJob, type OverwriteJobResult, type OverwriteWriterCatalogConfig, type PyIcebergJob, createIcebergOverwriteWriter, createLocalIcebergSink, deleteSiteFromShard, httpBackend, overwriteWriterAsSink, subprocessBackend };
@@ -1,4 +1,4 @@
1
- import { ICEBERG_SCHEMAS } from "./_chunks/schema2.mjs";
1
+ import { ICEBERG_SCHEMAS, assertIcebergTable } from "./_chunks/schema2.mjs";
2
2
  import { dirname, join } from "node:path";
3
3
  import { fileURLToPath } from "node:url";
4
4
  import process from "node:process";
@@ -83,4 +83,102 @@ function createLocalIcebergSink(options) {
83
83
  }
84
84
  };
85
85
  }
86
- export { createLocalIcebergSink };
86
+ function createIcebergOverwriteWriter(opts) {
87
+ const { catalog, backend } = opts;
88
+ async function overwriteSlice(slice, rows) {
89
+ const siteId = slice.ctx.siteId;
90
+ if (!siteId) throw new Error("overwriteSlice: slice.ctx.siteId is required for the Iceberg partition key");
91
+ const table = assertIcebergTable(slice.table);
92
+ const res = await backend({
93
+ op: "overwrite",
94
+ catalogUri: catalog.catalogUri,
95
+ namespace: catalog.namespace,
96
+ warehouse: catalog.warehouse,
97
+ s3: catalog.s3,
98
+ table,
99
+ spec: ICEBERG_SCHEMAS[table],
100
+ siteId,
101
+ searchType: slice.searchType,
102
+ date: slice.date,
103
+ rows
104
+ });
105
+ if (res.error) throw new Error(`overwriteSlice: PyIceberg backend failed: ${res.error}`);
106
+ return { rowCount: res.rowCount ?? rows.length };
107
+ }
108
+ return {
109
+ overwriteSlice,
110
+ async close() {
111
+ return {
112
+ flushed: [],
113
+ failed: []
114
+ };
115
+ }
116
+ };
117
+ }
118
+ function subprocessBackend(opts = {}) {
119
+ return async (job) => {
120
+ const { dirname, join } = await import("node:path");
121
+ const { fileURLToPath } = await import("node:url");
122
+ return runPyIcebergWriter({
123
+ python: resolvePyIcebergPython(opts.python),
124
+ script: opts.writerScript ?? join(dirname(fileURLToPath(import.meta.url)), "..", "scripts", "iceberg-writer.py"),
125
+ job,
126
+ label: "iceberg overwrite subprocess",
127
+ processErrorAsParseFailure: true
128
+ });
129
+ };
130
+ }
131
+ function httpBackend(opts) {
132
+ const doFetch = opts.fetch ?? fetch;
133
+ return async (job) => {
134
+ const res = await doFetch(opts.endpoint, {
135
+ method: "POST",
136
+ headers: {
137
+ "content-type": "application/json",
138
+ ...opts.token ? { authorization: `Bearer ${opts.token}` } : {}
139
+ },
140
+ body: JSON.stringify(job)
141
+ });
142
+ if (!res.ok) {
143
+ const body = await res.text().catch(() => "");
144
+ return { error: `HTTP ${res.status}${body ? `: ${body}` : ""}` };
145
+ }
146
+ return await res.json();
147
+ };
148
+ }
149
+ function overwriteWriterAsSink(writer) {
150
+ return {
151
+ capabilities: {
152
+ canOverwrite: true,
153
+ appendOnly: false
154
+ },
155
+ emit: (slice, rows) => writer.overwriteSlice(slice, rows),
156
+ overwriteSlice: (slice, rows) => writer.overwriteSlice(slice, rows),
157
+ close: () => writer.close()
158
+ };
159
+ }
160
+ async function deleteSiteFromShard(args) {
161
+ const { catalog, backend, siteId, tables } = args;
162
+ const results = [];
163
+ for (const table of tables) {
164
+ const res = await backend({
165
+ op: "delete",
166
+ catalogUri: catalog.catalogUri,
167
+ namespace: catalog.namespace,
168
+ warehouse: catalog.warehouse,
169
+ s3: catalog.s3,
170
+ ...catalog.catalogToken ? { catalogToken: catalog.catalogToken } : {},
171
+ table,
172
+ siteId
173
+ }).catch((err) => ({ error: err instanceof Error ? err.message : String(err) }));
174
+ results.push(res.error ? {
175
+ table,
176
+ error: res.error
177
+ } : {
178
+ table,
179
+ rowCount: res.rowCount
180
+ });
181
+ }
182
+ return results;
183
+ }
184
+ export { createIcebergOverwriteWriter, createLocalIcebergSink, deleteSiteFromShard, httpBackend, overwriteWriterAsSink, subprocessBackend };
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@gscdump/engine",
3
3
  "type": "module",
4
- "version": "0.32.6",
4
+ "version": "0.32.8",
5
5
  "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
6
6
  "author": {
7
7
  "name": "Harlan Wilton",
@@ -181,8 +181,8 @@
181
181
  "dependencies": {
182
182
  "drizzle-orm": "1.0.0-rc.3",
183
183
  "proper-lockfile": "^4.1.2",
184
- "@gscdump/contracts": "0.32.6",
185
- "gscdump": "0.32.6"
184
+ "gscdump": "0.32.8",
185
+ "@gscdump/contracts": "0.32.8"
186
186
  },
187
187
  "devDependencies": {
188
188
  "@duckdb/duckdb-wasm": "^1.32.0",