@gscdump/engine 0.32.6 → 0.32.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_chunks/libs/icebird.d.mts +3 -1
- package/dist/_chunks/libs/icebird.mjs +6 -4
- package/dist/_chunks/sink.d.mts +25 -10
- package/dist/iceberg/index.mjs +24 -1
- package/dist/sink-node.d.mts +161 -2
- package/dist/sink-node.mjs +100 -2
- package/package.json +3 -3
|
@@ -303,7 +303,8 @@ declare function icebergAppend({
|
|
|
303
303
|
tableUrl,
|
|
304
304
|
resolver,
|
|
305
305
|
records,
|
|
306
|
-
sortOrderId
|
|
306
|
+
sortOrderId,
|
|
307
|
+
snapshotProperties
|
|
307
308
|
}: {
|
|
308
309
|
catalog: Catalog;
|
|
309
310
|
namespace?: string | string[] | undefined;
|
|
@@ -312,6 +313,7 @@ declare function icebergAppend({
|
|
|
312
313
|
resolver?: Resolver | undefined;
|
|
313
314
|
records: Record<string, any>[];
|
|
314
315
|
sortOrderId?: number | undefined;
|
|
316
|
+
snapshotProperties?: Record<string, string> | undefined;
|
|
315
317
|
}): Promise<TableMetadata>;
|
|
316
318
|
/**
|
|
317
319
|
* Create a new table. REST: delegates to the catalog's create endpoint.
|
|
@@ -3216,7 +3216,7 @@ async function prepareAppend({ tableUrl, metadata, records, resolver, sortOrderI
|
|
|
3216
3216
|
writtenFiles: [...writtenDataFiles.map((f) => f.path), manifestPath]
|
|
3217
3217
|
};
|
|
3218
3218
|
}
|
|
3219
|
-
async function stageSnapshotForAppend({ tableUrl, metadata, prepared, resolver }) {
|
|
3219
|
+
async function stageSnapshotForAppend({ tableUrl, metadata, prepared, resolver, snapshotProperties }) {
|
|
3220
3220
|
if (!tableUrl) throw new Error("tableUrl is required");
|
|
3221
3221
|
if (!resolver?.writer) throw new Error("resolver.writer is required");
|
|
3222
3222
|
const sequenceNumber = BigInt(metadata["last-sequence-number"] ?? 0) + 1n;
|
|
@@ -3254,7 +3254,8 @@ async function stageSnapshotForAppend({ tableUrl, metadata, prepared, resolver }
|
|
|
3254
3254
|
"total-data-files": String(prevTotals.files + BigInt(prepared.addedDataFilesCount)),
|
|
3255
3255
|
"total-delete-files": "0",
|
|
3256
3256
|
"total-position-deletes": "0",
|
|
3257
|
-
"total-equality-deletes": "0"
|
|
3257
|
+
"total-equality-deletes": "0",
|
|
3258
|
+
...snapshotProperties ?? {}
|
|
3258
3259
|
};
|
|
3259
3260
|
return await buildSnapshotUpdate({
|
|
3260
3261
|
tableUrl,
|
|
@@ -3370,7 +3371,7 @@ const DEFAULT_RETRY = Object.freeze({
|
|
|
3370
3371
|
factor: 2,
|
|
3371
3372
|
totalTimeoutMs: 1800 * 1e3
|
|
3372
3373
|
});
|
|
3373
|
-
async function icebergAppend({ catalog, namespace, table, tableUrl, resolver, records, sortOrderId }) {
|
|
3374
|
+
async function icebergAppend({ catalog, namespace, table, tableUrl, resolver, records, sortOrderId, snapshotProperties }) {
|
|
3374
3375
|
const ctx = await loadTable({
|
|
3375
3376
|
catalog,
|
|
3376
3377
|
namespace,
|
|
@@ -3396,7 +3397,8 @@ async function icebergAppend({ catalog, namespace, table, tableUrl, resolver, re
|
|
|
3396
3397
|
tableUrl: workingCtx.tableUrl,
|
|
3397
3398
|
metadata: workingCtx.metadata,
|
|
3398
3399
|
prepared,
|
|
3399
|
-
resolver: requireResolver(workingCtx.resolver, "icebergAppend")
|
|
3400
|
+
resolver: requireResolver(workingCtx.resolver, "icebergAppend"),
|
|
3401
|
+
snapshotProperties
|
|
3400
3402
|
})
|
|
3401
3403
|
});
|
|
3402
3404
|
}
|
package/dist/_chunks/sink.d.mts
CHANGED
|
@@ -283,6 +283,14 @@ interface CommitRetryOptions {
|
|
|
283
283
|
sleep?: (ms: number) => Promise<void>;
|
|
284
284
|
/** Injectable RNG for the jitter — tests pass a deterministic value. */
|
|
285
285
|
random?: () => number;
|
|
286
|
+
/**
|
|
287
|
+
* Idempotency token stamped into the appended snapshot's summary
|
|
288
|
+
* (`gscdump.append-id`) and matched by the landed-check before any outer
|
|
289
|
+
* retry. Generated per call (`crypto.randomUUID`) so it is STABLE across this
|
|
290
|
+
* call's retries but UNIQUE per call — a content hash would risk a false
|
|
291
|
+
* "already landed" skip (silent data loss). Injectable for deterministic tests.
|
|
292
|
+
*/
|
|
293
|
+
appendId?: string;
|
|
286
294
|
}
|
|
287
295
|
/**
|
|
288
296
|
* True when `err` is an R2 Data Catalog commit rate-limit response
|
|
@@ -297,15 +305,18 @@ declare function isCommitRateLimited(err: unknown): boolean;
|
|
|
297
305
|
* retries 412/409 internally; 429 is the gap this closes. Non-429 errors
|
|
298
306
|
* (and 429s that survive every attempt) propagate unchanged.
|
|
299
307
|
*
|
|
300
|
-
*
|
|
301
|
-
*
|
|
302
|
-
*
|
|
303
|
-
*
|
|
304
|
-
*
|
|
305
|
-
*
|
|
306
|
-
*
|
|
307
|
-
*
|
|
308
|
-
*
|
|
308
|
+
* IDEMPOTENT RE-RUN. A 429 that escapes icebird's internal loop is retried HERE
|
|
309
|
+
* by re-running the whole `icebergAppend`, which re-prepares + re-uploads the
|
|
310
|
+
* data files. That is safe ONLY if the prior attempt's commit did NOT land — but
|
|
311
|
+
* R2 Data Catalog can apply a commit and STILL return 429 (rate-limit on the
|
|
312
|
+
* response path), and a lost ack looks identical. A blind re-run then appends a
|
|
313
|
+
* SECOND copy of the same rows → silent double-count (observed: a re-resynced
|
|
314
|
+
* site read 2× its true totals). To close this, every attempt stamps a per-call
|
|
315
|
+
* `gscdump.append-id` into the snapshot summary, and BEFORE retrying or giving
|
|
316
|
+
* up we reload the table and check whether that id already landed
|
|
317
|
+
* ({@link appendAlreadyLanded}); if so the append succeeded and we return without
|
|
318
|
+
* re-appending. A non-landed 429 still re-runs (the previous attempt's parquet is
|
|
319
|
+
* an orphan, reclaimed by R2 cleanup) — same as before, no double.
|
|
309
320
|
*/
|
|
310
321
|
declare function icebergAppendRetrying(args: Parameters<typeof icebergAppend>[0], options?: CommitRetryOptions): Promise<void>;
|
|
311
322
|
/**
|
|
@@ -443,6 +454,10 @@ interface SinkCapabilities {
|
|
|
443
454
|
/** Whether the sink exposes partition overwrite semantics. */
|
|
444
455
|
canOverwrite?: boolean;
|
|
445
456
|
}
|
|
457
|
+
/** Partition-overwrite writer for revision paths. */
|
|
458
|
+
interface SliceOverwriteWriter {
|
|
459
|
+
overwriteSlice: (slice: SinkSlice, rows: readonly Row$1[]) => Promise<SinkWriteResult>;
|
|
460
|
+
}
|
|
446
461
|
/**
|
|
447
462
|
* Outcome of `Sink.close()` — which tables' buffered rows reached durable
|
|
448
463
|
* storage and which failed.
|
|
@@ -528,4 +543,4 @@ interface LocalIcebergSinkOptions extends SinkOptions {
|
|
|
528
543
|
/** S3-compatible warehouse location (POC: MinIO). */
|
|
529
544
|
warehouse: string;
|
|
530
545
|
}
|
|
531
|
-
export { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, LocalIcebergSinkOptions, PartitionKeyEncoding, SEARCH_TYPE_INT, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };
|
|
546
|
+
export { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, LocalIcebergSinkOptions, PartitionKeyEncoding, SEARCH_TYPE_INT, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, SliceOverwriteWriter, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };
|
package/dist/iceberg/index.mjs
CHANGED
|
@@ -173,6 +173,8 @@ async function connectIcebergCatalog(config, opts = {}) {
|
|
|
173
173
|
namespace: config.namespace
|
|
174
174
|
};
|
|
175
175
|
}
|
|
176
|
+
const APPEND_ID_SUMMARY_KEY = "gscdump.append-id";
|
|
177
|
+
const APPEND_LANDED_SCAN_DEPTH = 25;
|
|
176
178
|
function isCommitRateLimited(err) {
|
|
177
179
|
if (err && typeof err === "object" && err.status === 429) return true;
|
|
178
180
|
const msg = (err instanceof Error ? err.message : String(err)).toLowerCase();
|
|
@@ -187,14 +189,35 @@ async function icebergAppendRetrying(args, options = {}) {
|
|
|
187
189
|
const maxDelayMs = options.maxDelayMs ?? 2e4;
|
|
188
190
|
const sleep = options.sleep ?? defaultCommitSleep;
|
|
189
191
|
const random = options.random ?? Math.random;
|
|
192
|
+
const appendId = options.appendId ?? globalThis.crypto.randomUUID();
|
|
193
|
+
const stampedArgs = {
|
|
194
|
+
...args,
|
|
195
|
+
snapshotProperties: {
|
|
196
|
+
...args.snapshotProperties,
|
|
197
|
+
[APPEND_ID_SUMMARY_KEY]: appendId
|
|
198
|
+
}
|
|
199
|
+
};
|
|
190
200
|
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
|
191
|
-
const err = await icebergAppend(
|
|
201
|
+
const err = await icebergAppend(stampedArgs).then(() => void 0, (e) => e);
|
|
192
202
|
if (err === void 0) return;
|
|
203
|
+
if (await appendAlreadyLanded(args, appendId).catch(() => false)) return;
|
|
193
204
|
if (!isCommitRateLimited(err) || attempt === maxAttempts - 1) throw err;
|
|
194
205
|
const ceiling = Math.min(maxDelayMs, baseDelayMs * 2 ** attempt);
|
|
195
206
|
await sleep(Math.floor(random() * ceiling));
|
|
196
207
|
}
|
|
197
208
|
}
|
|
209
|
+
async function appendAlreadyLanded(args, appendId) {
|
|
210
|
+
const a = args;
|
|
211
|
+
if (a.catalog?.type !== "rest" || a.namespace == null || a.table == null) return false;
|
|
212
|
+
const { metadata } = await restCatalogLoadTable(a.catalog, {
|
|
213
|
+
namespace: a.namespace,
|
|
214
|
+
table: a.table
|
|
215
|
+
});
|
|
216
|
+
const snapshots = metadata.snapshots ?? [];
|
|
217
|
+
const from = Math.max(0, snapshots.length - APPEND_LANDED_SCAN_DEPTH);
|
|
218
|
+
for (let i = snapshots.length - 1; i >= from; i--) if (snapshots[i]?.summary?.[APPEND_ID_SUMMARY_KEY] === appendId) return true;
|
|
219
|
+
return false;
|
|
220
|
+
}
|
|
198
221
|
async function ensureIcebergNamespace(conn) {
|
|
199
222
|
await restCatalogCreateNamespace(conn.catalog, { namespace: conn.namespace }).catch(() => {});
|
|
200
223
|
}
|
package/dist/sink-node.d.mts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Row } from "./_chunks/storage.mjs";
|
|
2
|
+
import { IcebergS3Config, LocalIcebergSinkOptions, Sink, SinkCloseResult, SliceOverwriteWriter } from "./_chunks/sink.mjs";
|
|
2
3
|
/** Full `LocalIcebergSink` options — extends the frozen contract options. */
|
|
3
4
|
interface LocalIcebergSinkFullOptions extends LocalIcebergSinkOptions {
|
|
4
5
|
/** S3 credentials for the warehouse. Defaults to the POC MinIO creds. */
|
|
@@ -21,4 +22,162 @@ interface LocalIcebergSink extends Sink {
|
|
|
21
22
|
* use this sink must skip when the stack is unreachable.
|
|
22
23
|
*/
|
|
23
24
|
declare function createLocalIcebergSink(options: LocalIcebergSinkFullOptions): LocalIcebergSink;
|
|
24
|
-
|
|
25
|
+
/** Connection details for the Iceberg REST catalog the writer targets. */
|
|
26
|
+
interface OverwriteWriterCatalogConfig {
|
|
27
|
+
/**
|
|
28
|
+
* Iceberg REST catalog endpoint, e.g. `http://localhost:8181` (POC) or the
|
|
29
|
+
* R2 Data Catalog endpoint (prod).
|
|
30
|
+
*/
|
|
31
|
+
catalogUri: string;
|
|
32
|
+
/** Catalog namespace the 5 fact tables live under, e.g. `gsc`. */
|
|
33
|
+
namespace: string;
|
|
34
|
+
/** Warehouse identifier the catalog resolves table locations under. */
|
|
35
|
+
warehouse: string;
|
|
36
|
+
/** S3-compatible object store backing the catalog. */
|
|
37
|
+
s3: IcebergS3Config;
|
|
38
|
+
/** Bearer token for a token-authed REST catalog (R2 Data Catalog). */
|
|
39
|
+
catalogToken?: string;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* The JSON job wire-format shared with `scripts/iceberg-writer.py` — identical
|
|
43
|
+
* to the shape `LocalIcebergSink` already emits, so one PyIceberg script backs
|
|
44
|
+
* both the local sink and this writer. `op` is always `overwrite` here.
|
|
45
|
+
*/
|
|
46
|
+
interface OverwriteJob {
|
|
47
|
+
op: 'overwrite';
|
|
48
|
+
catalogUri: string;
|
|
49
|
+
namespace: string;
|
|
50
|
+
warehouse: string;
|
|
51
|
+
s3: IcebergS3Config;
|
|
52
|
+
table: string;
|
|
53
|
+
spec: unknown;
|
|
54
|
+
siteId: string;
|
|
55
|
+
searchType: string;
|
|
56
|
+
date: string;
|
|
57
|
+
/** Data columns only — the script injects `site_id` / `search_type`. */
|
|
58
|
+
rows: readonly Row[];
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Per-site delete job — removes EVERY row for one `siteId` across a whole
|
|
62
|
+
* table (all dates / search types). Recovery op for clearing a corrupt site
|
|
63
|
+
* (e.g. a cross-run append double) from a shared per-team shard before a clean
|
|
64
|
+
* re-backfill, without touching sibling sites. Carries no rows. `siteId` is a
|
|
65
|
+
* `number` for an INT-encoded shard, a `string` for STRING encoding.
|
|
66
|
+
*/
|
|
67
|
+
interface DeleteJob {
|
|
68
|
+
op: 'delete';
|
|
69
|
+
catalogUri: string;
|
|
70
|
+
namespace: string;
|
|
71
|
+
warehouse: string;
|
|
72
|
+
s3: IcebergS3Config;
|
|
73
|
+
/** Bearer token for a token-authed REST catalog (R2 Data Catalog). */
|
|
74
|
+
catalogToken?: string;
|
|
75
|
+
table: string;
|
|
76
|
+
siteId: string | number;
|
|
77
|
+
}
|
|
78
|
+
/** Any job the shared PyIceberg backend can execute. */
|
|
79
|
+
type PyIcebergJob = OverwriteJob | DeleteJob;
|
|
80
|
+
/** Result wire-format from the PyIceberg backend. */
|
|
81
|
+
interface OverwriteJobResult {
|
|
82
|
+
/** Rows written (emit/overwrite) or removed (delete). */
|
|
83
|
+
rowCount?: number;
|
|
84
|
+
error?: string;
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* A transport that executes one PyIceberg job and returns its result.
|
|
88
|
+
* `subprocessBackend` and `httpBackend` are the two shipped implementations;
|
|
89
|
+
* tests inject a fake. The param is the `OverwriteJob | DeleteJob` union, so a
|
|
90
|
+
* backend value is still assignable where the narrower `OverwriteBackend` is
|
|
91
|
+
* expected (a wider-param function accepts the narrower call).
|
|
92
|
+
*/
|
|
93
|
+
type OverwriteBackend = (job: PyIcebergJob) => Promise<OverwriteJobResult>;
|
|
94
|
+
interface IcebergOverwriteWriterOptions {
|
|
95
|
+
catalog: OverwriteWriterCatalogConfig;
|
|
96
|
+
/** The transport that runs the PyIceberg job. */
|
|
97
|
+
backend: OverwriteBackend;
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* The trailing-window Iceberg overwrite writer. Conforms to the frozen
|
|
101
|
+
* `SliceOverwriteWriter` contract; `overwriteSlice` builds one PyIceberg
|
|
102
|
+
* overwrite job per call and runs it through the configured backend.
|
|
103
|
+
*/
|
|
104
|
+
interface IcebergOverwriteWriter extends SliceOverwriteWriter {
|
|
105
|
+
/** Release any backend resources. Idempotent. */
|
|
106
|
+
close: () => Promise<SinkCloseResult>;
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Build a trailing-window Iceberg overwrite writer over the given backend.
|
|
110
|
+
* Pure — all I/O is the backend's; the writer only shapes the job.
|
|
111
|
+
*/
|
|
112
|
+
declare function createIcebergOverwriteWriter(opts: IcebergOverwriteWriterOptions): IcebergOverwriteWriter;
|
|
113
|
+
/** Options for the PyIceberg subprocess backend. */
|
|
114
|
+
interface SubprocessBackendOptions {
|
|
115
|
+
/** Python interpreter. Defaults to `$GSCDUMP_ICEBERG_PYTHON` then `python3`. */
|
|
116
|
+
python?: string;
|
|
117
|
+
/**
|
|
118
|
+
* Path to the PyIceberg writer script. Defaults to
|
|
119
|
+
* `<engine>/scripts/iceberg-writer.py`.
|
|
120
|
+
*/
|
|
121
|
+
writerScript?: string;
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Backend that runs the PyIceberg overwrite by spawning the shared writer
|
|
125
|
+
* script (`scripts/iceberg-writer.py`). Node-only — used by local integration
|
|
126
|
+
* tests and a Node-side scheduled job box. The script reads the job JSON on
|
|
127
|
+
* stdin and writes `{rowCount}` / `{error}` on stdout.
|
|
128
|
+
*/
|
|
129
|
+
declare function subprocessBackend(opts?: SubprocessBackendOptions): OverwriteBackend;
|
|
130
|
+
/** Options for the PyIceberg HTTP backend (the Cloudflare Container in prod). */
|
|
131
|
+
interface HttpBackendOptions {
|
|
132
|
+
/**
|
|
133
|
+
* URL of the PyIceberg overwrite service — a POST endpoint accepting the
|
|
134
|
+
* `OverwriteJob` JSON and returning `OverwriteJobResult`.
|
|
135
|
+
*/
|
|
136
|
+
endpoint: string;
|
|
137
|
+
/** Optional bearer token for the service. */
|
|
138
|
+
token?: string;
|
|
139
|
+
/** Injectable fetch — defaults to the global. */
|
|
140
|
+
fetch?: typeof fetch;
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Backend that POSTs the overwrite job to a PyIceberg HTTP service — the
|
|
144
|
+
* Cloudflare Container running `iceberg-writer.py` behind an HTTP shim. This
|
|
145
|
+
* is the prod transport: the request-path Worker holds no Python and never
|
|
146
|
+
* does an Iceberg metadata commit itself.
|
|
147
|
+
*/
|
|
148
|
+
declare function httpBackend(opts: HttpBackendOptions): OverwriteBackend;
|
|
149
|
+
/**
|
|
150
|
+
* Adapt an `IcebergOverwriteWriter` into a full `Sink` whose `emit` is also a
|
|
151
|
+
* partition-overwrite — convenient for tests so a single Iceberg writer backs
|
|
152
|
+
* both append and revision paths against the local stack. Prod keeps `emit`
|
|
153
|
+
* on the Pipeline; only `overwriteSlice` delegates here.
|
|
154
|
+
*/
|
|
155
|
+
declare function overwriteWriterAsSink(writer: IcebergOverwriteWriter): Sink & SliceOverwriteWriter;
|
|
156
|
+
/** Per-table outcome of a {@link deleteSiteFromShard} run. */
|
|
157
|
+
interface DeleteSiteResult {
|
|
158
|
+
table: string;
|
|
159
|
+
/** Rows removed, when the backend reported a count. */
|
|
160
|
+
rowCount?: number;
|
|
161
|
+
/** Present when this table's delete failed; the sweep continues regardless. */
|
|
162
|
+
error?: string;
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Remove every row for one `siteId` from a per-team Iceberg shard, across the
|
|
166
|
+
* given tables. Recovery op: clear a corrupt site (e.g. a cross-run append
|
|
167
|
+
* double) before a clean re-backfill, leaving sibling sites in the shared shard
|
|
168
|
+
* untouched. `site_id` is an identity partition column, so PyIceberg prunes
|
|
169
|
+
* whole data files (no row-level delete files).
|
|
170
|
+
*
|
|
171
|
+
* Runs SEQUENTIALLY (one commit per table at a time) so the per-table catalog
|
|
172
|
+
* commit-rate ceiling is never tripped. A table failure is captured in its
|
|
173
|
+
* result row, not thrown — the sweep always attempts every table so a partial
|
|
174
|
+
* failure is visible and re-runnable (delete is idempotent: re-deleting an
|
|
175
|
+
* already-empty site is a no-op).
|
|
176
|
+
*/
|
|
177
|
+
declare function deleteSiteFromShard(args: {
|
|
178
|
+
catalog: OverwriteWriterCatalogConfig;
|
|
179
|
+
backend: OverwriteBackend;
|
|
180
|
+
siteId: string | number;
|
|
181
|
+
tables: readonly string[];
|
|
182
|
+
}): Promise<DeleteSiteResult[]>;
|
|
183
|
+
export { type DeleteJob, type DeleteSiteResult, type IcebergOverwriteWriter, type IcebergOverwriteWriterOptions, type LocalIcebergSink, type LocalIcebergSinkFullOptions, type LocalIcebergSinkOptions, type OverwriteBackend, type OverwriteJob, type OverwriteJobResult, type OverwriteWriterCatalogConfig, type PyIcebergJob, createIcebergOverwriteWriter, createLocalIcebergSink, deleteSiteFromShard, httpBackend, overwriteWriterAsSink, subprocessBackend };
|
package/dist/sink-node.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { ICEBERG_SCHEMAS } from "./_chunks/schema2.mjs";
|
|
1
|
+
import { ICEBERG_SCHEMAS, assertIcebergTable } from "./_chunks/schema2.mjs";
|
|
2
2
|
import { dirname, join } from "node:path";
|
|
3
3
|
import { fileURLToPath } from "node:url";
|
|
4
4
|
import process from "node:process";
|
|
@@ -83,4 +83,102 @@ function createLocalIcebergSink(options) {
|
|
|
83
83
|
}
|
|
84
84
|
};
|
|
85
85
|
}
|
|
86
|
-
|
|
86
|
+
function createIcebergOverwriteWriter(opts) {
|
|
87
|
+
const { catalog, backend } = opts;
|
|
88
|
+
async function overwriteSlice(slice, rows) {
|
|
89
|
+
const siteId = slice.ctx.siteId;
|
|
90
|
+
if (!siteId) throw new Error("overwriteSlice: slice.ctx.siteId is required for the Iceberg partition key");
|
|
91
|
+
const table = assertIcebergTable(slice.table);
|
|
92
|
+
const res = await backend({
|
|
93
|
+
op: "overwrite",
|
|
94
|
+
catalogUri: catalog.catalogUri,
|
|
95
|
+
namespace: catalog.namespace,
|
|
96
|
+
warehouse: catalog.warehouse,
|
|
97
|
+
s3: catalog.s3,
|
|
98
|
+
table,
|
|
99
|
+
spec: ICEBERG_SCHEMAS[table],
|
|
100
|
+
siteId,
|
|
101
|
+
searchType: slice.searchType,
|
|
102
|
+
date: slice.date,
|
|
103
|
+
rows
|
|
104
|
+
});
|
|
105
|
+
if (res.error) throw new Error(`overwriteSlice: PyIceberg backend failed: ${res.error}`);
|
|
106
|
+
return { rowCount: res.rowCount ?? rows.length };
|
|
107
|
+
}
|
|
108
|
+
return {
|
|
109
|
+
overwriteSlice,
|
|
110
|
+
async close() {
|
|
111
|
+
return {
|
|
112
|
+
flushed: [],
|
|
113
|
+
failed: []
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
function subprocessBackend(opts = {}) {
|
|
119
|
+
return async (job) => {
|
|
120
|
+
const { dirname, join } = await import("node:path");
|
|
121
|
+
const { fileURLToPath } = await import("node:url");
|
|
122
|
+
return runPyIcebergWriter({
|
|
123
|
+
python: resolvePyIcebergPython(opts.python),
|
|
124
|
+
script: opts.writerScript ?? join(dirname(fileURLToPath(import.meta.url)), "..", "scripts", "iceberg-writer.py"),
|
|
125
|
+
job,
|
|
126
|
+
label: "iceberg overwrite subprocess",
|
|
127
|
+
processErrorAsParseFailure: true
|
|
128
|
+
});
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
function httpBackend(opts) {
|
|
132
|
+
const doFetch = opts.fetch ?? fetch;
|
|
133
|
+
return async (job) => {
|
|
134
|
+
const res = await doFetch(opts.endpoint, {
|
|
135
|
+
method: "POST",
|
|
136
|
+
headers: {
|
|
137
|
+
"content-type": "application/json",
|
|
138
|
+
...opts.token ? { authorization: `Bearer ${opts.token}` } : {}
|
|
139
|
+
},
|
|
140
|
+
body: JSON.stringify(job)
|
|
141
|
+
});
|
|
142
|
+
if (!res.ok) {
|
|
143
|
+
const body = await res.text().catch(() => "");
|
|
144
|
+
return { error: `HTTP ${res.status}${body ? `: ${body}` : ""}` };
|
|
145
|
+
}
|
|
146
|
+
return await res.json();
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
function overwriteWriterAsSink(writer) {
|
|
150
|
+
return {
|
|
151
|
+
capabilities: {
|
|
152
|
+
canOverwrite: true,
|
|
153
|
+
appendOnly: false
|
|
154
|
+
},
|
|
155
|
+
emit: (slice, rows) => writer.overwriteSlice(slice, rows),
|
|
156
|
+
overwriteSlice: (slice, rows) => writer.overwriteSlice(slice, rows),
|
|
157
|
+
close: () => writer.close()
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
async function deleteSiteFromShard(args) {
|
|
161
|
+
const { catalog, backend, siteId, tables } = args;
|
|
162
|
+
const results = [];
|
|
163
|
+
for (const table of tables) {
|
|
164
|
+
const res = await backend({
|
|
165
|
+
op: "delete",
|
|
166
|
+
catalogUri: catalog.catalogUri,
|
|
167
|
+
namespace: catalog.namespace,
|
|
168
|
+
warehouse: catalog.warehouse,
|
|
169
|
+
s3: catalog.s3,
|
|
170
|
+
...catalog.catalogToken ? { catalogToken: catalog.catalogToken } : {},
|
|
171
|
+
table,
|
|
172
|
+
siteId
|
|
173
|
+
}).catch((err) => ({ error: err instanceof Error ? err.message : String(err) }));
|
|
174
|
+
results.push(res.error ? {
|
|
175
|
+
table,
|
|
176
|
+
error: res.error
|
|
177
|
+
} : {
|
|
178
|
+
table,
|
|
179
|
+
rowCount: res.rowCount
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
return results;
|
|
183
|
+
}
|
|
184
|
+
export { createIcebergOverwriteWriter, createLocalIcebergSink, deleteSiteFromShard, httpBackend, overwriteWriterAsSink, subprocessBackend };
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gscdump/engine",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "0.32.
|
|
4
|
+
"version": "0.32.8",
|
|
5
5
|
"description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -181,8 +181,8 @@
|
|
|
181
181
|
"dependencies": {
|
|
182
182
|
"drizzle-orm": "1.0.0-rc.3",
|
|
183
183
|
"proper-lockfile": "^4.1.2",
|
|
184
|
-
"
|
|
185
|
-
"gscdump": "0.32.
|
|
184
|
+
"gscdump": "0.32.8",
|
|
185
|
+
"@gscdump/contracts": "0.32.8"
|
|
186
186
|
},
|
|
187
187
|
"devDependencies": {
|
|
188
188
|
"@duckdb/duckdb-wasm": "^1.32.0",
|