@gscdump/engine 1.4.5 → 1.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/entities.d.mts +11 -4
- package/dist/entities.mjs +111 -10
- package/dist/entity-keys.d.mts +12 -1
- package/dist/entity-keys.mjs +7 -1
- package/package.json +4 -4
package/dist/entities.d.mts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { DataSource } from "./storage.mjs";
|
|
2
2
|
import { ScheduleState } from "./schedule.mjs";
|
|
3
|
-
import { SitemapGenerationKey, emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, parseSitemapUrlsDeltaKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsEventKey, sitemapUrlsEventSeedKey, sitemapUrlsEventsPrefix, sitemapUrlsGenerationKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPendingGenerationKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey, sitemapUrlsReconcileGenerationKey } from "./entity-keys.mjs";
|
|
4
|
-
import { SITEMAP_PROJECTION_GRACE_MS, SitemapProjectionFeed, SitemapProjectionFiles, SitemapProjectionManifest, decodeSitemapProjectionManifest, emptySitemapProjectionManifest, parseSitemapProjectionManifest, selectSitemapProjectionFiles, withSitemapProjectionFeed } from "./sitemap-projection.mjs";
|
|
3
|
+
import { SitemapGenerationKey, emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapUrlsDeltaKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsEventKey, sitemapUrlsEventSeedKey, sitemapUrlsEventsPrefix, sitemapUrlsGenerationKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPendingGenerationKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey, sitemapUrlsReconcileGenerationKey } from "./entity-keys.mjs";
|
|
5
4
|
import { QueryDimDeps, QueryDimMeta, QueryDimRecord, QueryDimStore, buildQueryDimRecords, createQueryDimStore } from "./query-dim.mjs";
|
|
5
|
+
import { SITEMAP_PROJECTION_GRACE_MS, SitemapProjectionFeed, SitemapProjectionFiles, SitemapProjectionManifest, decodeSitemapProjectionManifest, emptySitemapProjectionManifest, parseSitemapProjectionManifest, selectSitemapProjectionFiles, withSitemapProjectionFeed } from "./sitemap-projection.mjs";
|
|
6
6
|
import { TenantCtx } from "@gscdump/contracts";
|
|
7
7
|
/**
|
|
8
8
|
* GSC URL inspection result fields we persist. Mirrors the
|
|
@@ -200,10 +200,17 @@ interface InspectionStore {
|
|
|
200
200
|
*
|
|
201
201
|
* No-op (no base rewrite) when there are zero outstanding events.
|
|
202
202
|
*/
|
|
203
|
-
compactInspections: (ctx: TenantCtx
|
|
203
|
+
compactInspections: (ctx: TenantCtx, opts?: {
|
|
204
|
+
/**
|
|
205
|
+
* Also record state changes into the durable transitions sidecar.
|
|
206
|
+
* Default OFF so publishing this is inert until a canary opts in.
|
|
207
|
+
*/
|
|
208
|
+
transitions?: boolean;
|
|
209
|
+
}) => Promise<{
|
|
204
210
|
baseRowCount: number;
|
|
205
211
|
eventsFolded: number;
|
|
206
212
|
eventFilesDeleted: number;
|
|
213
|
+
transitionsWritten: number;
|
|
207
214
|
}>;
|
|
208
215
|
/**
|
|
209
216
|
* DuckDB-resolvable URI for the materialised parquet sidecar, or
|
|
@@ -460,4 +467,4 @@ interface CreateEmptyTypesStoreOptions {
|
|
|
460
467
|
now?: () => number;
|
|
461
468
|
}
|
|
462
469
|
declare function createEmptyTypesStore(opts: CreateEmptyTypesStoreOptions): EmptyTypesStore;
|
|
463
|
-
export { CompactUrlsOptions, CompactUrlsResult, CompleteSitemapGeneration, CreateEmptyTypesStoreOptions, CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, CreateSitemapReadStoreOptions, CreateSitemapStoreOptions, DateRange, DeltaEntry, EmptyTypesDoc, EmptyTypesStore, INSPECTION_HISTORY_MAX_BYTES, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, InspectionEventRow, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, LoadUrlsOptions, ParsedUrl, QueryDimDeps, QueryDimMeta, QueryDimRecord, QueryDimStore, ReconcileResult, SITEMAP_PROJECTION_GRACE_MS, SitemapGenerationKey, SitemapHistoryDoc, SitemapIndex, SitemapMembershipEvent, SitemapMutation, SitemapProjectionFeed, SitemapProjectionFiles, SitemapProjectionManifest, SitemapReadStore, SitemapRecord, SitemapStore, SitemapUrlRecord, SnapshotUrlsResult, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapReadStore, createSitemapStore, decodeSitemapProjectionManifest, emptySitemapProjectionManifest, emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, parseSitemapProjectionManifest, parseSitemapUrlsDeltaKey, queryDimMetaKey, queryDimParquetKey, selectSitemapProjectionFiles, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsEventKey, sitemapUrlsEventSeedKey, sitemapUrlsEventsPrefix, sitemapUrlsGenerationKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPendingGenerationKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey, sitemapUrlsReconcileGenerationKey, withSitemapProjectionFeed };
|
|
470
|
+
export { CompactUrlsOptions, CompactUrlsResult, CompleteSitemapGeneration, CreateEmptyTypesStoreOptions, CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, CreateSitemapReadStoreOptions, CreateSitemapStoreOptions, DateRange, DeltaEntry, EmptyTypesDoc, EmptyTypesStore, INSPECTION_HISTORY_MAX_BYTES, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, InspectionEventRow, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, LoadUrlsOptions, ParsedUrl, QueryDimDeps, QueryDimMeta, QueryDimRecord, QueryDimStore, ReconcileResult, SITEMAP_PROJECTION_GRACE_MS, SitemapGenerationKey, SitemapHistoryDoc, SitemapIndex, SitemapMembershipEvent, SitemapMutation, SitemapProjectionFeed, SitemapProjectionFiles, SitemapProjectionManifest, SitemapReadStore, SitemapRecord, SitemapStore, SitemapUrlRecord, SnapshotUrlsResult, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapReadStore, createSitemapStore, decodeSitemapProjectionManifest, emptySitemapProjectionManifest, emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapProjectionManifest, parseSitemapUrlsDeltaKey, queryDimMetaKey, queryDimParquetKey, selectSitemapProjectionFiles, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsEventKey, sitemapUrlsEventSeedKey, sitemapUrlsEventsPrefix, sitemapUrlsGenerationKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPendingGenerationKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey, sitemapUrlsReconcileGenerationKey, withSitemapProjectionFeed };
|
package/dist/entities.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, parseSitemapUrlsDeltaKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsEventKey, sitemapUrlsEventSeedKey, sitemapUrlsEventsPrefix, sitemapUrlsGenerationKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPendingGenerationKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey, sitemapUrlsReconcileGenerationKey } from "./entity-keys.mjs";
|
|
1
|
+
import { emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapUrlsDeltaKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsEventKey, sitemapUrlsEventSeedKey, sitemapUrlsEventsPrefix, sitemapUrlsGenerationKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPendingGenerationKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey, sitemapUrlsReconcileGenerationKey } from "./entity-keys.mjs";
|
|
2
2
|
import { decodeParquetToRows, encodeRowsToParquetFlex } from "./adapters/hyparquet.mjs";
|
|
3
3
|
import { readOptional } from "./adapters/read-optional.mjs";
|
|
4
4
|
import { SITEMAP_PROJECTION_GRACE_MS, decodeSitemapProjectionManifest, emptySitemapProjectionManifest, parseSitemapProjectionManifest, selectSitemapProjectionFiles, withSitemapProjectionFeed } from "./sitemap-projection.mjs";
|
|
@@ -157,6 +157,99 @@ const INSPECTION_EVENT_COLUMNS = [
|
|
|
157
157
|
nullable: true
|
|
158
158
|
}
|
|
159
159
|
];
|
|
160
|
+
const TRANSITION_STATE_FIELDS = [
|
|
161
|
+
"indexStatus",
|
|
162
|
+
"coverageState",
|
|
163
|
+
"robotsTxtState",
|
|
164
|
+
"indexingState",
|
|
165
|
+
"pageFetchState",
|
|
166
|
+
"googleCanonical"
|
|
167
|
+
];
|
|
168
|
+
const INSPECTION_TRANSITION_COLUMNS = [
|
|
169
|
+
{
|
|
170
|
+
name: "urlHash",
|
|
171
|
+
type: "VARCHAR",
|
|
172
|
+
nullable: false
|
|
173
|
+
},
|
|
174
|
+
{
|
|
175
|
+
name: "url",
|
|
176
|
+
type: "VARCHAR",
|
|
177
|
+
nullable: false
|
|
178
|
+
},
|
|
179
|
+
{
|
|
180
|
+
name: "changedAfter",
|
|
181
|
+
type: "VARCHAR",
|
|
182
|
+
nullable: false
|
|
183
|
+
},
|
|
184
|
+
{
|
|
185
|
+
name: "changedBefore",
|
|
186
|
+
type: "VARCHAR",
|
|
187
|
+
nullable: false
|
|
188
|
+
},
|
|
189
|
+
...TRANSITION_STATE_FIELDS.flatMap((field) => {
|
|
190
|
+
const capped = field[0].toUpperCase() + field.slice(1);
|
|
191
|
+
return [{
|
|
192
|
+
name: `from${capped}`,
|
|
193
|
+
type: "VARCHAR",
|
|
194
|
+
nullable: true
|
|
195
|
+
}, {
|
|
196
|
+
name: `to${capped}`,
|
|
197
|
+
type: "VARCHAR",
|
|
198
|
+
nullable: true
|
|
199
|
+
}];
|
|
200
|
+
})
|
|
201
|
+
];
|
|
202
|
+
function isStateTransition(before, after) {
|
|
203
|
+
return TRANSITION_STATE_FIELDS.some((field) => (before[field] ?? null) !== (after[field] ?? null));
|
|
204
|
+
}
|
|
205
|
+
function buildTransitionRow(before, after) {
|
|
206
|
+
const row = {
|
|
207
|
+
urlHash: String(after.urlHash),
|
|
208
|
+
url: String(after.url ?? before.url ?? ""),
|
|
209
|
+
changedAfter: String(before.inspectedAt ?? ""),
|
|
210
|
+
changedBefore: String(after.inspectedAt ?? "")
|
|
211
|
+
};
|
|
212
|
+
for (const field of TRANSITION_STATE_FIELDS) {
|
|
213
|
+
const capped = field[0].toUpperCase() + field.slice(1);
|
|
214
|
+
row[`from${capped}`] = before[field] ?? null;
|
|
215
|
+
row[`to${capped}`] = after[field] ?? null;
|
|
216
|
+
}
|
|
217
|
+
return row;
|
|
218
|
+
}
|
|
219
|
+
async function appendTransitions(ds, ctx, rows) {
|
|
220
|
+
if (rows.length === 0) return 0;
|
|
221
|
+
const byMonth = /* @__PURE__ */ new Map();
|
|
222
|
+
for (const row of rows) {
|
|
223
|
+
const month = transitionMonth(row);
|
|
224
|
+
const bucket = byMonth.get(month);
|
|
225
|
+
if (bucket) bucket.push(row);
|
|
226
|
+
else byMonth.set(month, [row]);
|
|
227
|
+
}
|
|
228
|
+
let written = 0;
|
|
229
|
+
for (const [month, monthRows] of byMonth) {
|
|
230
|
+
const key = inspectionTransitionsMonthKey(ctx, month);
|
|
231
|
+
const existingBytes = await readOptional(ds, key);
|
|
232
|
+
const existing = existingBytes ? await decodeParquetToRows(existingBytes) : [];
|
|
233
|
+
const seen = new Set(existing.map((r) => `${String(r.urlHash)}\u0000${String(r.changedBefore)}`));
|
|
234
|
+
const fresh = monthRows.filter((r) => {
|
|
235
|
+
const id = `${String(r.urlHash)}\u0000${String(r.changedBefore)}`;
|
|
236
|
+
if (seen.has(id)) return false;
|
|
237
|
+
seen.add(id);
|
|
238
|
+
return true;
|
|
239
|
+
});
|
|
240
|
+
if (fresh.length === 0) continue;
|
|
241
|
+
const merged = [...existing, ...fresh];
|
|
242
|
+
await ds.write(key, encodeRowsToParquetFlex(merged, {
|
|
243
|
+
columns: INSPECTION_TRANSITION_COLUMNS,
|
|
244
|
+
sortKey: ["urlHash"]
|
|
245
|
+
}));
|
|
246
|
+
written += fresh.length;
|
|
247
|
+
}
|
|
248
|
+
return written;
|
|
249
|
+
}
|
|
250
|
+
function transitionMonth(row) {
|
|
251
|
+
return String(row.changedBefore ?? "").slice(0, 7) || "unknown";
|
|
252
|
+
}
|
|
160
253
|
function createInspectionStore(opts) {
|
|
161
254
|
const ds = opts.dataSource;
|
|
162
255
|
function shardFor(record) {
|
|
@@ -250,22 +343,27 @@ function createInspectionStore(opts) {
|
|
|
250
343
|
rowCount: rows.length
|
|
251
344
|
};
|
|
252
345
|
},
|
|
253
|
-
async compactInspections(ctx) {
|
|
346
|
+
async compactInspections(ctx, opts) {
|
|
254
347
|
const eventKeys = (await ds.list(`${inspectionEventsPrefix(ctx)}/`)).filter((k) => INSPECTION_EVENT_KEY_RE.test(k));
|
|
255
348
|
if (eventKeys.length === 0) return {
|
|
256
349
|
baseRowCount: 0,
|
|
257
350
|
eventsFolded: 0,
|
|
258
|
-
eventFilesDeleted: 0
|
|
351
|
+
eventFilesDeleted: 0,
|
|
352
|
+
transitionsWritten: 0
|
|
259
353
|
};
|
|
260
354
|
const baseKey = inspectionBaseKey(ctx);
|
|
261
355
|
const baseBytes = await readOptional(ds, baseKey);
|
|
262
356
|
const baseRows = baseBytes ? await decodeParquetToRows(baseBytes) : [];
|
|
263
357
|
const latest = /* @__PURE__ */ new Map();
|
|
264
358
|
const earliestChecked = /* @__PURE__ */ new Map();
|
|
359
|
+
const transitions = [];
|
|
265
360
|
const consider = (row) => {
|
|
266
361
|
const h = String(row.urlHash);
|
|
267
362
|
const prev = latest.get(h);
|
|
268
|
-
if (!prev || String(row.inspectedAt ?? "") > String(prev.inspectedAt ?? ""))
|
|
363
|
+
if (!prev || String(row.inspectedAt ?? "") > String(prev.inspectedAt ?? "")) {
|
|
364
|
+
if (opts?.transitions && prev && isStateTransition(prev, row)) transitions.push(buildTransitionRow(prev, row));
|
|
365
|
+
latest.set(h, row);
|
|
366
|
+
}
|
|
269
367
|
const fc = row.firstCheckedAt;
|
|
270
368
|
if (fc != null) {
|
|
271
369
|
const fcStr = String(fc);
|
|
@@ -276,6 +374,7 @@ function createInspectionStore(opts) {
|
|
|
276
374
|
for (const row of baseRows) consider(row);
|
|
277
375
|
let eventsFolded = 0;
|
|
278
376
|
const consumed = [];
|
|
377
|
+
const eventRows = [];
|
|
279
378
|
const eventFiles = await mapEntityIo(eventKeys.sort(), async (key) => {
|
|
280
379
|
const bytes = await readOptional(ds, key);
|
|
281
380
|
if (!bytes) return void 0;
|
|
@@ -288,11 +387,11 @@ function createInspectionStore(opts) {
|
|
|
288
387
|
if (!file) continue;
|
|
289
388
|
const { key, rows } = file;
|
|
290
389
|
consumed.push(key);
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
eventsFolded++;
|
|
294
|
-
}
|
|
390
|
+
eventRows.push(...rows);
|
|
391
|
+
eventsFolded += rows.length;
|
|
295
392
|
}
|
|
393
|
+
eventRows.sort((a, b) => String(a.urlHash).localeCompare(String(b.urlHash)) || String(a.inspectedAt ?? "").localeCompare(String(b.inspectedAt ?? "")));
|
|
394
|
+
for (const row of eventRows) consider(row);
|
|
296
395
|
const merged = [];
|
|
297
396
|
for (const [h, row] of latest) {
|
|
298
397
|
const fc = earliestChecked.get(h);
|
|
@@ -303,12 +402,14 @@ function createInspectionStore(opts) {
|
|
|
303
402
|
columns: INSPECTION_EVENT_COLUMNS,
|
|
304
403
|
sortKey: ["urlHash"]
|
|
305
404
|
});
|
|
405
|
+
const transitionsWritten = opts?.transitions ? await appendTransitions(ds, ctx, transitions) : 0;
|
|
306
406
|
await ds.write(baseKey, bytes);
|
|
307
407
|
if (consumed.length > 0) await ds.delete(consumed);
|
|
308
408
|
return {
|
|
309
409
|
baseRowCount: merged.length,
|
|
310
410
|
eventsFolded,
|
|
311
|
-
eventFilesDeleted: consumed.length
|
|
411
|
+
eventFilesDeleted: consumed.length,
|
|
412
|
+
transitionsWritten
|
|
312
413
|
};
|
|
313
414
|
},
|
|
314
415
|
parquetUri(ctx) {
|
|
@@ -1239,4 +1340,4 @@ function createEmptyTypesStore(opts) {
|
|
|
1239
1340
|
}
|
|
1240
1341
|
};
|
|
1241
1342
|
}
|
|
1242
|
-
export { INSPECTION_HISTORY_MAX_BYTES, SITEMAP_PROJECTION_GRACE_MS, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapReadStore, createSitemapStore, decodeSitemapProjectionManifest, emptySitemapProjectionManifest, emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, parseSitemapProjectionManifest, parseSitemapUrlsDeltaKey, queryDimMetaKey, queryDimParquetKey, selectSitemapProjectionFiles, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsEventKey, sitemapUrlsEventSeedKey, sitemapUrlsEventsPrefix, sitemapUrlsGenerationKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPendingGenerationKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey, sitemapUrlsReconcileGenerationKey, withSitemapProjectionFeed };
|
|
1343
|
+
export { INSPECTION_HISTORY_MAX_BYTES, SITEMAP_PROJECTION_GRACE_MS, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapReadStore, createSitemapStore, decodeSitemapProjectionManifest, emptySitemapProjectionManifest, emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapProjectionManifest, parseSitemapUrlsDeltaKey, queryDimMetaKey, queryDimParquetKey, selectSitemapProjectionFiles, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsEventKey, sitemapUrlsEventSeedKey, sitemapUrlsEventsPrefix, sitemapUrlsGenerationKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPendingGenerationKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey, sitemapUrlsReconcileGenerationKey, withSitemapProjectionFeed };
|
package/dist/entity-keys.d.mts
CHANGED
|
@@ -5,6 +5,17 @@ declare function inspectionParquetKey(ctx: TenantCtx): string;
|
|
|
5
5
|
declare function inspectionEventsPrefix(ctx: TenantCtx): string;
|
|
6
6
|
declare function inspectionEventKey(ctx: TenantCtx, yearMonth: string, batchId: string): string;
|
|
7
7
|
declare function inspectionBaseKey(ctx: TenantCtx): string;
|
|
8
|
+
/**
|
|
9
|
+
* Append-only record of URL-inspection STATE CHANGES, partitioned by month.
|
|
10
|
+
*
|
|
11
|
+
* Distinct from `events/`, which compaction deletes. Google exposes no API for
|
|
12
|
+
* historical index-coverage state, so a verdict change observed once is gone
|
|
13
|
+
* the moment the event file folds — this is the only durable record that a URL
|
|
14
|
+
* ever left (or entered) the index.
|
|
15
|
+
*/
|
|
16
|
+
declare function inspectionTransitionsPrefix(ctx: TenantCtx): string;
|
|
17
|
+
/** One file per calendar month (`YYYY-MM`), rewritten in place as it fills. */
|
|
18
|
+
declare function inspectionTransitionsMonthKey(ctx: TenantCtx, yearMonth: string): string;
|
|
8
19
|
declare function inspectionHistoryPrefix(ctx: TenantCtx, yearMonth: string): string;
|
|
9
20
|
declare function inspectionHistoryShardKey(ctx: TenantCtx, yearMonth: string, batchId: string): string;
|
|
10
21
|
/** Stable, URL-safe FNV-1a hash used for entity keys. */
|
|
@@ -40,4 +51,4 @@ declare function hashSortedUrlList(locs: readonly string[]): string;
|
|
|
40
51
|
declare function indexingMetadataIndexKey(ctx: TenantCtx): string;
|
|
41
52
|
declare function queryDimParquetKey(ctx: TenantCtx): string;
|
|
42
53
|
declare function queryDimMetaKey(ctx: TenantCtx): string;
|
|
43
|
-
export { SitemapGenerationKey, emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, parseSitemapUrlsDeltaKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsEventKey, sitemapUrlsEventSeedKey, sitemapUrlsEventsPrefix, sitemapUrlsGenerationKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPendingGenerationKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey, sitemapUrlsReconcileGenerationKey };
|
|
54
|
+
export { SitemapGenerationKey, emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapUrlsDeltaKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsEventKey, sitemapUrlsEventSeedKey, sitemapUrlsEventsPrefix, sitemapUrlsGenerationKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPendingGenerationKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey, sitemapUrlsReconcileGenerationKey };
|
package/dist/entity-keys.mjs
CHANGED
|
@@ -19,6 +19,12 @@ function inspectionEventKey(ctx, yearMonth, batchId) {
|
|
|
19
19
|
function inspectionBaseKey(ctx) {
|
|
20
20
|
return `${tenantEntityPrefix(ctx)}/inspections/base.parquet`;
|
|
21
21
|
}
|
|
22
|
+
function inspectionTransitionsPrefix(ctx) {
|
|
23
|
+
return `${tenantEntityPrefix(ctx)}/inspections/transitions`;
|
|
24
|
+
}
|
|
25
|
+
function inspectionTransitionsMonthKey(ctx, yearMonth) {
|
|
26
|
+
return `${inspectionTransitionsPrefix(ctx)}/${yearMonth}.parquet`;
|
|
27
|
+
}
|
|
22
28
|
function inspectionHistoryPrefix(ctx, yearMonth) {
|
|
23
29
|
return `${tenantEntityPrefix(ctx)}/inspections/history/${yearMonth}`;
|
|
24
30
|
}
|
|
@@ -125,4 +131,4 @@ function queryDimParquetKey(ctx) {
|
|
|
125
131
|
function queryDimMetaKey(ctx) {
|
|
126
132
|
return `${queryDimPrefix(ctx)}/index.json`;
|
|
127
133
|
}
|
|
128
|
-
export { emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, parseSitemapUrlsDeltaKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsEventKey, sitemapUrlsEventSeedKey, sitemapUrlsEventsPrefix, sitemapUrlsGenerationKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPendingGenerationKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey, sitemapUrlsReconcileGenerationKey };
|
|
134
|
+
export { emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapUrlsDeltaKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsEventKey, sitemapUrlsEventSeedKey, sitemapUrlsEventsPrefix, sitemapUrlsGenerationKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPendingGenerationKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey, sitemapUrlsReconcileGenerationKey };
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gscdump/engine",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "1.4.
|
|
4
|
+
"version": "1.4.7",
|
|
5
5
|
"description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -181,9 +181,9 @@
|
|
|
181
181
|
"dependencies": {
|
|
182
182
|
"drizzle-orm": "1.0.0-rc.3",
|
|
183
183
|
"proper-lockfile": "^4.1.2",
|
|
184
|
-
"@gscdump/contracts": "^1.4.
|
|
185
|
-
"@gscdump/lakehouse": "^1.4.
|
|
186
|
-
"gscdump": "^1.4.
|
|
184
|
+
"@gscdump/contracts": "^1.4.7",
|
|
185
|
+
"@gscdump/lakehouse": "^1.4.7",
|
|
186
|
+
"gscdump": "^1.4.7"
|
|
187
187
|
},
|
|
188
188
|
"devDependencies": {
|
|
189
189
|
"@duckdb/duckdb-wasm": "1.33.1-dev57.0",
|