@gscdump/engine 0.40.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/dist/_chunks/duckdb.d.mts +2 -9
- package/dist/_chunks/engine.mjs +3 -7
- package/dist/_chunks/entities.mjs +1 -1
- package/dist/_chunks/index.d.mts +3 -8
- package/dist/_chunks/index2.d.mts +1 -2
- package/dist/_chunks/libs/hyparquet-compressors.mjs +1 -1
- package/dist/_chunks/libs/hyparquet.mjs +1 -63
- package/dist/_chunks/libs/icebird.d.mts +1 -81
- package/dist/_chunks/libs/icebird.mjs +4 -330
- package/dist/_chunks/parquet-plan.mjs +248 -3
- package/dist/_chunks/resolver.mjs +16 -2
- package/dist/_chunks/schema.d.mts +2 -2
- package/dist/_chunks/sink.d.mts +4 -5
- package/dist/_chunks/source.mjs +3 -3
- package/dist/_chunks/storage.d.mts +1 -43
- package/dist/_chunks/types.d.mts +8 -0
- package/dist/adapters/node.d.mts +1 -11
- package/dist/adapters/node.mjs +1 -1
- package/dist/analyzer/index.d.mts +1 -1
- package/dist/entities.d.mts +2 -10
- package/dist/entities.mjs +2 -2
- package/dist/iceberg/index.d.mts +2 -3
- package/dist/iceberg/index.mjs +12 -12
- package/dist/index.d.mts +83 -2
- package/dist/index.mjs +365 -6
- package/dist/period/index.d.mts +2 -2
- package/dist/period/index.mjs +1 -1
- package/dist/planner.mjs +1 -2
- package/dist/resolver/index.d.mts +2 -9
- package/dist/resolver/index.mjs +2 -2
- package/dist/rollups.d.mts +3 -20
- package/dist/rollups.mjs +5 -5
- package/dist/scope.d.mts +1 -2
- package/dist/scope.mjs +1 -1
- package/dist/source/index.d.mts +2 -2
- package/dist/source/index.mjs +2 -2
- package/dist/sql-bind.d.mts +1 -29
- package/dist/sql-bind.mjs +1 -1
- package/package.json +7 -22
- package/dist/_chunks/compaction.mjs +0 -251
- package/dist/adapters/r2-manifest.d.mts +0 -82
- package/dist/adapters/r2-manifest.mjs +0 -364
- package/dist/compaction-public.d.mts +0 -15
- package/dist/compaction-public.mjs +0 -5
- package/dist/snapshot.d.mts +0 -2
- package/dist/snapshot.mjs +0 -1
package/dist/rollups.mjs
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import "./_chunks/layout.mjs";
|
|
2
1
|
import { engineErrors } from "./errors.mjs";
|
|
2
|
+
import "./_chunks/layout.mjs";
|
|
3
3
|
import { encodeRowsToParquetFlex } from "./adapters/hyparquet.mjs";
|
|
4
4
|
import { createIndexingMetadataStore, createQueryDimStore, createSitemapStore, inspectionParquetKey, sitemapUrlsIndexPrefix } from "./_chunks/entities.mjs";
|
|
5
|
-
import { encodeJsonBigintSafe } from "@gscdump/lakehouse";
|
|
6
5
|
import { MS_PER_DAY } from "gscdump/dates";
|
|
6
|
+
import { encodeJsonBigintSafe } from "@gscdump/lakehouse";
|
|
7
7
|
function rollupPrefix(ctx, searchType) {
|
|
8
8
|
const base = ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/rollups` : `u_${ctx.userId}/rollups`;
|
|
9
9
|
return searchType !== void 0 && searchType !== "web" ? `${base}/${searchType}` : base;
|
|
@@ -754,7 +754,7 @@ const queryCanonicalDailyRollup = {
|
|
|
754
754
|
orderBy: "date, query_canonical",
|
|
755
755
|
pageRows: ROLLUP_PAGE_ROWS_DAILY
|
|
756
756
|
},
|
|
757
|
-
maxWindowDays:
|
|
757
|
+
maxWindowDays: DAILY_MAX_WINDOW_DAYS,
|
|
758
758
|
sqlFor: dailyWindowSqlFor(useDim, canonExpr)
|
|
759
759
|
})).map(mapDailyRow);
|
|
760
760
|
}
|
|
@@ -800,7 +800,7 @@ async function rebuildCanonicalDailyResumable(opts) {
|
|
|
800
800
|
const sType = searchType !== void 0 ? { searchType } : {};
|
|
801
801
|
const startPageOffset = opts.pageOffset ?? 0;
|
|
802
802
|
const pageRows = opts.pageRows ?? 7e4;
|
|
803
|
-
const maxWindowDays = opts.maxWindowDays ??
|
|
803
|
+
const maxWindowDays = opts.maxWindowDays ?? DAILY_MAX_WINDOW_DAYS;
|
|
804
804
|
const shardCount = Math.max(1, Math.floor(opts.shardCount ?? 1));
|
|
805
805
|
const windows = planRollupWindows((await engine.listPartitions({
|
|
806
806
|
ctx,
|
|
@@ -1261,4 +1261,4 @@ const DEFAULT_ROLLUPS = [
|
|
|
1261
1261
|
sitemapChanges28dRollup
|
|
1262
1262
|
];
|
|
1263
1263
|
const CANONICAL_ROLLUPS = [queryCanonicalVariantsRollup, queryCanonicalDailyRollup];
|
|
1264
|
-
export { CANONICAL_ROLLUPS,
|
|
1264
|
+
export { CANONICAL_ROLLUPS, DEFAULT_ROLLUPS, ROLLUP_PAGE_ROWS, ROLLUP_PAGE_ROWS_DAILY, ROLLUP_PAGE_ROWS_WIDE, WINDOW_BYTE_BUDGET, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, planRollupWindows, queryCanonicalDailyRollup, queryCanonicalVariantsRollup, readLatestRollup, rebuildCanonicalDailyResumable, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, runWindowed, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topPages28dRollup, weeklyTotalsRollup };
|
package/dist/scope.d.mts
CHANGED
|
@@ -28,7 +28,6 @@ interface TableScope {
|
|
|
28
28
|
window?: ResolvedWindow;
|
|
29
29
|
siteId?: string;
|
|
30
30
|
}
|
|
31
|
-
declare function buildTableScope(table: Record<string, any>, opts: ScopedRunnerOptions): TableScope;
|
|
32
31
|
declare function mergeScope(scope: TableScope, ...extra: SQL[]): SQL | undefined;
|
|
33
32
|
/**
|
|
34
33
|
* Bind `buildTableScope` + `mergeScope` to a specific drizzle schema. Engine
|
|
@@ -41,4 +40,4 @@ declare function createScopedHelpers<S extends Record<string, Record<string, any
|
|
|
41
40
|
scopeFor: (table: keyof S, opts: ScopedRunnerOptions) => TableScope;
|
|
42
41
|
mergeScope: typeof mergeScope;
|
|
43
42
|
};
|
|
44
|
-
export { ResolvedWindow, ScopedRunnerOptions, TableScope,
|
|
43
|
+
export { ResolvedWindow, ScopedRunnerOptions, TableScope, createScopedHelpers, mergeScope };
|
package/dist/scope.mjs
CHANGED
package/dist/source/index.d.mts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import { AnalysisQuerySource, AnalysisSourceKind, ExecuteSqlOptions, FileSet, QueryRow, SourceCapabilities } from "../_chunks/registry.mjs";
|
|
2
|
-
import { AttachedTableMissingError, AttachedTableRunner, AttachedTableSourceOptions, CreateSqlQuerySourceOptions, ENGINE_QUERY_CAPABILITIES, EngineQuerySourceOptions, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows,
|
|
3
|
-
export { type AnalysisQuerySource, type AnalysisSourceKind, AttachedTableMissingError, type AttachedTableRunner, type AttachedTableSourceOptions, type CreateSqlQuerySourceOptions, ENGINE_QUERY_CAPABILITIES, EngineQuerySourceOptions, type ExecuteSqlOptions, type FileSet, type QueryRow, type SourceCapabilities, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows,
|
|
2
|
+
import { AttachedTableMissingError, AttachedTableRunner, AttachedTableSourceOptions, CreateSqlQuerySourceOptions, ENGINE_QUERY_CAPABILITIES, EngineQuerySourceOptions, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows, runAnalyzerWithEngine } from "../_chunks/index.mjs";
|
|
3
|
+
export { type AnalysisQuerySource, type AnalysisSourceKind, AttachedTableMissingError, type AttachedTableRunner, type AttachedTableSourceOptions, type CreateSqlQuerySourceOptions, ENGINE_QUERY_CAPABILITIES, EngineQuerySourceOptions, type ExecuteSqlOptions, type FileSet, type QueryRow, type SourceCapabilities, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows, runAnalyzerWithEngine };
|
package/dist/source/index.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { AttachedTableMissingError, ENGINE_QUERY_CAPABILITIES, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows,
|
|
2
|
-
export { AttachedTableMissingError, ENGINE_QUERY_CAPABILITIES, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows,
|
|
1
|
+
import { AttachedTableMissingError, ENGINE_QUERY_CAPABILITIES, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows, runAnalyzerWithEngine } from "../_chunks/source.mjs";
|
|
2
|
+
export { AttachedTableMissingError, ENGINE_QUERY_CAPABILITIES, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows, runAnalyzerWithEngine };
|
package/dist/sql-bind.d.mts
CHANGED
|
@@ -1,33 +1,5 @@
|
|
|
1
|
-
import { EngineError } from "./_chunks/errors.mjs";
|
|
2
|
-
import { Result } from "gscdump/result";
|
|
3
1
|
/** Escape single quotes for inlining inside a SQL string literal (SQL-standard `''` escaping). */
|
|
4
2
|
declare function sqlEscape(s: string): string;
|
|
5
|
-
/**
|
|
6
|
-
* Errors-as-values core for {@link formatLiteral}: returns a typed
|
|
7
|
-
* `invalid-sql-literal` `EngineError` for values that can't be safely inlined.
|
|
8
|
-
*/
|
|
9
|
-
declare function formatLiteralResult(value: unknown): Result<string, EngineError>;
|
|
10
3
|
declare function formatLiteral(value: unknown): string;
|
|
11
|
-
/**
|
|
12
|
-
* Replace `?` and `$N` placeholders with inline SQL literals. Single-quoted
|
|
13
|
-
* string regions and SQL comments (`-- line`, `/* block *\/`) are left
|
|
14
|
-
* untouched — a `?` or `$1` inside `'foo?bar'` or a comment is not a
|
|
15
|
-
* placeholder. SQL-standard `''` escape handling; no `\`-escape or
|
|
16
|
-
* dialect-specific identifier quoting.
|
|
17
|
-
*
|
|
18
|
-
* `?` placeholders bind sequentially against `params`. `$N` (Postgres-style)
|
|
19
|
-
* binds explicitly to `params[N-1]`. The two styles must not be mixed in the
|
|
20
|
-
* same query.
|
|
21
|
-
*
|
|
22
|
-
* Throws when placeholder count and params length disagree, or when a `$N`
|
|
23
|
-
* index is out of range.
|
|
24
|
-
*/
|
|
25
|
-
/**
|
|
26
|
-
* Errors-as-values core for {@link bindLiterals}: returns a typed
|
|
27
|
-
* `placeholder-arity-mismatch` / `invalid-sql-literal` `EngineError` instead of
|
|
28
|
-
* throwing, so the edge RPC / proxy callers that build SQL from untrusted params
|
|
29
|
-
* can branch on the failure.
|
|
30
|
-
*/
|
|
31
|
-
declare function bindLiteralsResult(sql: string, params: readonly unknown[]): Result<string, EngineError>;
|
|
32
4
|
declare function bindLiterals(sql: string, params: readonly unknown[]): string;
|
|
33
|
-
export { bindLiterals,
|
|
5
|
+
export { bindLiterals, formatLiteral, sqlEscape };
|
package/dist/sql-bind.mjs
CHANGED
|
@@ -101,4 +101,4 @@ function bindLiteralsResult(sql, params) {
|
|
|
101
101
|
function bindLiterals(sql, params) {
|
|
102
102
|
return unwrapResult(bindLiteralsResult(sql, params), engineErrorToException);
|
|
103
103
|
}
|
|
104
|
-
export { bindLiterals,
|
|
104
|
+
export { bindLiterals, formatLiteral, sqlEscape };
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gscdump/engine",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "1.0.0",
|
|
5
5
|
"description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -36,21 +36,11 @@
|
|
|
36
36
|
"import": "./dist/errors.mjs",
|
|
37
37
|
"default": "./dist/errors.mjs"
|
|
38
38
|
},
|
|
39
|
-
"./snapshot": {
|
|
40
|
-
"types": "./dist/snapshot.d.mts",
|
|
41
|
-
"import": "./dist/snapshot.mjs",
|
|
42
|
-
"default": "./dist/snapshot.mjs"
|
|
43
|
-
},
|
|
44
39
|
"./planner": {
|
|
45
40
|
"types": "./dist/planner.d.mts",
|
|
46
41
|
"import": "./dist/planner.mjs",
|
|
47
42
|
"default": "./dist/planner.mjs"
|
|
48
43
|
},
|
|
49
|
-
"./compaction": {
|
|
50
|
-
"types": "./dist/compaction-public.d.mts",
|
|
51
|
-
"import": "./dist/compaction-public.mjs",
|
|
52
|
-
"default": "./dist/compaction-public.mjs"
|
|
53
|
-
},
|
|
54
44
|
"./schema": {
|
|
55
45
|
"types": "./dist/schema.d.mts",
|
|
56
46
|
"import": "./dist/schema.mjs",
|
|
@@ -111,11 +101,6 @@
|
|
|
111
101
|
"import": "./dist/adapters/r2.mjs",
|
|
112
102
|
"default": "./dist/adapters/r2.mjs"
|
|
113
103
|
},
|
|
114
|
-
"./r2-manifest": {
|
|
115
|
-
"types": "./dist/adapters/r2-manifest.d.mts",
|
|
116
|
-
"import": "./dist/adapters/r2-manifest.mjs",
|
|
117
|
-
"default": "./dist/adapters/r2-manifest.mjs"
|
|
118
|
-
},
|
|
119
104
|
"./resolver": {
|
|
120
105
|
"types": "./dist/resolver/index.d.mts",
|
|
121
106
|
"import": "./dist/resolver/index.mjs",
|
|
@@ -168,10 +153,10 @@
|
|
|
168
153
|
"dist"
|
|
169
154
|
],
|
|
170
155
|
"engines": {
|
|
171
|
-
"node": ">=
|
|
156
|
+
"node": ">=22"
|
|
172
157
|
},
|
|
173
158
|
"peerDependencies": {
|
|
174
|
-
"@duckdb/duckdb-wasm": "^1.
|
|
159
|
+
"@duckdb/duckdb-wasm": "^1.33.1-dev57.0"
|
|
175
160
|
},
|
|
176
161
|
"peerDependenciesMeta": {
|
|
177
162
|
"@duckdb/duckdb-wasm": {
|
|
@@ -181,12 +166,12 @@
|
|
|
181
166
|
"dependencies": {
|
|
182
167
|
"drizzle-orm": "1.0.0-rc.3",
|
|
183
168
|
"proper-lockfile": "^4.1.2",
|
|
184
|
-
"@gscdump/
|
|
185
|
-
"
|
|
186
|
-
"gscdump": "0.
|
|
169
|
+
"@gscdump/lakehouse": "^1.0.0",
|
|
170
|
+
"gscdump": "^1.0.0",
|
|
171
|
+
"@gscdump/contracts": "^1.0.0"
|
|
187
172
|
},
|
|
188
173
|
"devDependencies": {
|
|
189
|
-
"@duckdb/duckdb-wasm": "
|
|
174
|
+
"@duckdb/duckdb-wasm": "1.33.1-dev57.0",
|
|
190
175
|
"@types/node": "^26.1.1",
|
|
191
176
|
"@types/proper-lockfile": "^4.1.4",
|
|
192
177
|
"aws4fetch": "^1.0.20",
|
|
@@ -1,251 +0,0 @@
|
|
|
1
|
-
import { dayPartition, inferSearchType, mondayOfWeek, monthPartition, objectKey, quarterOfMonth, quarterPartition, weekPartition } from "./layout.mjs";
|
|
2
|
-
import { currentSchemaVersion } from "./schema.mjs";
|
|
3
|
-
import { MS_PER_DAY } from "gscdump/dates";
|
|
4
|
-
const DAILY_PARTITION_RE = /^daily\/(\d{4}-\d{2}-\d{2})$/;
|
|
5
|
-
const WEEKLY_PARTITION_RE = /^weekly\/(\d{4}-\d{2}-\d{2})$/;
|
|
6
|
-
const MONTHLY_PARTITION_RE = /^monthly\/(\d{4}-\d{2})$/;
|
|
7
|
-
const QUARTERLY_PARTITION_RE = /^quarterly\/(\d{4})-Q([1-4])$/;
|
|
8
|
-
const DEFAULT_THRESHOLDS = {
|
|
9
|
-
raw: 7,
|
|
10
|
-
d7: 30,
|
|
11
|
-
d30: 90
|
|
12
|
-
};
|
|
13
|
-
function countRawDailies(entries) {
|
|
14
|
-
return entries.filter((e) => e.tier === "raw" || e.tier == null && e.partition.startsWith("daily/")).length;
|
|
15
|
-
}
|
|
16
|
-
const PENDING_WINDOW_DAYS = 4;
|
|
17
|
-
const STAGES = [
|
|
18
|
-
{
|
|
19
|
-
inputTier: "raw",
|
|
20
|
-
outputTier: "d7",
|
|
21
|
-
cutoffDays: DEFAULT_THRESHOLDS.raw,
|
|
22
|
-
bucketKey: (e) => {
|
|
23
|
-
const m = e.partition.match(DAILY_PARTITION_RE);
|
|
24
|
-
if (!m) return void 0;
|
|
25
|
-
return mondayOfWeek(m[1]);
|
|
26
|
-
},
|
|
27
|
-
bucketLatestMs: (monday) => Date.parse(`${monday}T00:00:00Z`) + 6 * MS_PER_DAY,
|
|
28
|
-
outputPartition: weekPartition
|
|
29
|
-
},
|
|
30
|
-
{
|
|
31
|
-
inputTier: "d7",
|
|
32
|
-
outputTier: "d30",
|
|
33
|
-
cutoffDays: DEFAULT_THRESHOLDS.d7,
|
|
34
|
-
bucketKey: (e) => {
|
|
35
|
-
const m = e.partition.match(WEEKLY_PARTITION_RE);
|
|
36
|
-
if (!m) return void 0;
|
|
37
|
-
return m[1].slice(0, 7);
|
|
38
|
-
},
|
|
39
|
-
bucketLatestMs: monthEndMs,
|
|
40
|
-
outputPartition: monthPartition
|
|
41
|
-
},
|
|
42
|
-
{
|
|
43
|
-
inputTier: "d30",
|
|
44
|
-
outputTier: "d90",
|
|
45
|
-
cutoffDays: DEFAULT_THRESHOLDS.d30,
|
|
46
|
-
bucketKey: (e) => {
|
|
47
|
-
const m = e.partition.match(MONTHLY_PARTITION_RE);
|
|
48
|
-
if (!m) return void 0;
|
|
49
|
-
return quarterOfMonth(m[1]);
|
|
50
|
-
},
|
|
51
|
-
bucketLatestMs: quarterEndMs,
|
|
52
|
-
outputPartition: quarterPartition
|
|
53
|
-
}
|
|
54
|
-
];
|
|
55
|
-
async function compactTieredImpl(deps, ctx, now, overrides = {}) {
|
|
56
|
-
const thresholds = {
|
|
57
|
-
...DEFAULT_THRESHOLDS,
|
|
58
|
-
...overrides
|
|
59
|
-
};
|
|
60
|
-
const stagesWithThresholds = STAGES.map((s) => ({
|
|
61
|
-
...s,
|
|
62
|
-
cutoffDays: s.outputTier === "d7" ? thresholds.raw : s.outputTier === "d30" ? thresholds.d7 : thresholds.d30
|
|
63
|
-
}));
|
|
64
|
-
for (const stage of stagesWithThresholds) await runStage(deps, ctx, stage, now);
|
|
65
|
-
}
|
|
66
|
-
async function runStage(deps, ctx, stage, now) {
|
|
67
|
-
const cutoff = now - Math.max(stage.cutoffDays, PENDING_WINDOW_DAYS) * MS_PER_DAY;
|
|
68
|
-
const candidates = await deps.manifestStore.listLive({
|
|
69
|
-
userId: ctx.userId,
|
|
70
|
-
siteId: ctx.siteId,
|
|
71
|
-
table: ctx.table,
|
|
72
|
-
tier: stage.inputTier
|
|
73
|
-
});
|
|
74
|
-
const buckets = /* @__PURE__ */ new Map();
|
|
75
|
-
for (const entry of candidates) {
|
|
76
|
-
if (entry.partition.startsWith("hourly/")) continue;
|
|
77
|
-
const key = stage.bucketKey(entry);
|
|
78
|
-
if (!key) continue;
|
|
79
|
-
if (stage.bucketLatestMs(key) >= cutoff) continue;
|
|
80
|
-
const compositeKey = `${inferSearchType(entry)}\0${key}`;
|
|
81
|
-
if (!buckets.has(compositeKey)) buckets.set(compositeKey, []);
|
|
82
|
-
buckets.get(compositeKey).push(entry);
|
|
83
|
-
}
|
|
84
|
-
for (const [compositeKey, entries] of buckets) {
|
|
85
|
-
const [searchType, bucket] = compositeKey.split("\0");
|
|
86
|
-
const targetPartition = stage.outputPartition(bucket);
|
|
87
|
-
if (entries.length === 1 && entries[0].partition === targetPartition) continue;
|
|
88
|
-
await deps.manifestStore.withLock({
|
|
89
|
-
userId: ctx.userId,
|
|
90
|
-
siteId: ctx.siteId,
|
|
91
|
-
table: ctx.table,
|
|
92
|
-
partition: targetPartition
|
|
93
|
-
}, async () => {
|
|
94
|
-
const key = objectKey(ctx, ctx.table, targetPartition, now, searchType);
|
|
95
|
-
const { bytes, rowCount } = await deps.codec.compactRows({ table: ctx.table }, entries.map((e) => e.objectKey), key, deps.dataSource);
|
|
96
|
-
const newEntry = {
|
|
97
|
-
userId: ctx.userId,
|
|
98
|
-
siteId: ctx.siteId,
|
|
99
|
-
table: ctx.table,
|
|
100
|
-
partition: targetPartition,
|
|
101
|
-
objectKey: key,
|
|
102
|
-
rowCount,
|
|
103
|
-
bytes,
|
|
104
|
-
createdAt: now,
|
|
105
|
-
schemaVersion: currentSchemaVersion(ctx.table),
|
|
106
|
-
tier: stage.outputTier,
|
|
107
|
-
...searchType !== "web" ? { searchType } : {}
|
|
108
|
-
};
|
|
109
|
-
await deps.manifestStore.registerVersion(newEntry, entries);
|
|
110
|
-
});
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
|
-
function enumeratePartitions(startDate, endDate) {
|
|
114
|
-
const out = [];
|
|
115
|
-
const [sy, sm, sd] = startDate.split("-").map(Number);
|
|
116
|
-
const [ey, em, ed] = endDate.split("-").map(Number);
|
|
117
|
-
const start = Date.UTC(sy, sm - 1, sd);
|
|
118
|
-
const end = Date.UTC(ey, em - 1, ed);
|
|
119
|
-
if (end < start) return out;
|
|
120
|
-
const seenWeeks = /* @__PURE__ */ new Set();
|
|
121
|
-
const seenMonths = /* @__PURE__ */ new Set();
|
|
122
|
-
const seenQuarters = /* @__PURE__ */ new Set();
|
|
123
|
-
for (let t = start; t <= end; t += 864e5) {
|
|
124
|
-
const d = new Date(t);
|
|
125
|
-
const y = d.getUTCFullYear();
|
|
126
|
-
const m = String(d.getUTCMonth() + 1).padStart(2, "0");
|
|
127
|
-
const isoDay = `${y}-${m}-${String(d.getUTCDate()).padStart(2, "0")}`;
|
|
128
|
-
const isoMonth = `${y}-${m}`;
|
|
129
|
-
out.push(dayPartition(isoDay));
|
|
130
|
-
const monday = mondayOfWeek(isoDay);
|
|
131
|
-
if (!seenWeeks.has(monday)) {
|
|
132
|
-
seenWeeks.add(monday);
|
|
133
|
-
out.push(weekPartition(monday));
|
|
134
|
-
}
|
|
135
|
-
if (!seenMonths.has(isoMonth)) {
|
|
136
|
-
seenMonths.add(isoMonth);
|
|
137
|
-
out.push(monthPartition(isoMonth));
|
|
138
|
-
}
|
|
139
|
-
const quarter = quarterOfMonth(isoMonth);
|
|
140
|
-
if (!seenQuarters.has(quarter)) {
|
|
141
|
-
seenQuarters.add(quarter);
|
|
142
|
-
out.push(quarterPartition(quarter));
|
|
143
|
-
}
|
|
144
|
-
}
|
|
145
|
-
return out;
|
|
146
|
-
}
|
|
147
|
-
function partitionSpan(partition) {
|
|
148
|
-
let m = partition.match(DAILY_PARTITION_RE);
|
|
149
|
-
if (m) {
|
|
150
|
-
const ms = Date.parse(`${m[1]}T00:00:00Z`);
|
|
151
|
-
return {
|
|
152
|
-
rank: 0,
|
|
153
|
-
startMs: ms,
|
|
154
|
-
endMs: ms
|
|
155
|
-
};
|
|
156
|
-
}
|
|
157
|
-
m = partition.match(WEEKLY_PARTITION_RE);
|
|
158
|
-
if (m) {
|
|
159
|
-
const ms = Date.parse(`${m[1]}T00:00:00Z`);
|
|
160
|
-
return {
|
|
161
|
-
rank: 1,
|
|
162
|
-
startMs: ms,
|
|
163
|
-
endMs: ms + 6 * MS_PER_DAY
|
|
164
|
-
};
|
|
165
|
-
}
|
|
166
|
-
m = partition.match(MONTHLY_PARTITION_RE);
|
|
167
|
-
if (m) {
|
|
168
|
-
const [y, mo] = m[1].split("-").map(Number);
|
|
169
|
-
return {
|
|
170
|
-
rank: 2,
|
|
171
|
-
startMs: Date.UTC(y, mo - 1, 1),
|
|
172
|
-
endMs: Date.UTC(y, mo, 0)
|
|
173
|
-
};
|
|
174
|
-
}
|
|
175
|
-
m = partition.match(QUARTERLY_PARTITION_RE);
|
|
176
|
-
if (m) {
|
|
177
|
-
const y = Number(m[1]);
|
|
178
|
-
const q = Number(m[2]);
|
|
179
|
-
return {
|
|
180
|
-
rank: 3,
|
|
181
|
-
startMs: Date.UTC(y, (q - 1) * 3, 1),
|
|
182
|
-
endMs: Date.UTC(y, q * 3, 0)
|
|
183
|
-
};
|
|
184
|
-
}
|
|
185
|
-
}
|
|
186
|
-
function splitOverlappingTiers(entries, queryRange) {
|
|
187
|
-
const rangeStartMs = queryRange ? Date.parse(`${queryRange.start}T00:00:00Z`) : void 0;
|
|
188
|
-
const rangeEndMs = queryRange ? Date.parse(`${queryRange.end}T00:00:00Z`) : void 0;
|
|
189
|
-
const spanned = [];
|
|
190
|
-
const kept = [];
|
|
191
|
-
const subsumed = [];
|
|
192
|
-
const clampToRange = Number.isFinite(rangeStartMs) && Number.isFinite(rangeEndMs);
|
|
193
|
-
for (const entry of entries) {
|
|
194
|
-
const span = partitionSpan(entry.partition);
|
|
195
|
-
if (!span) {
|
|
196
|
-
kept.push(entry);
|
|
197
|
-
continue;
|
|
198
|
-
}
|
|
199
|
-
const startMs = clampToRange ? Math.max(span.startMs, rangeStartMs) : span.startMs;
|
|
200
|
-
const endMs = clampToRange ? Math.min(span.endMs, rangeEndMs) : span.endMs;
|
|
201
|
-
if (queryRange && startMs > endMs) {
|
|
202
|
-
subsumed.push(entry);
|
|
203
|
-
continue;
|
|
204
|
-
}
|
|
205
|
-
spanned.push({
|
|
206
|
-
entry,
|
|
207
|
-
rank: span.rank,
|
|
208
|
-
startMs,
|
|
209
|
-
endMs
|
|
210
|
-
});
|
|
211
|
-
}
|
|
212
|
-
spanned.sort((a, b) => a.rank - b.rank || b.entry.createdAt - a.entry.createdAt);
|
|
213
|
-
const coveredBySearchType = /* @__PURE__ */ new Map();
|
|
214
|
-
for (const { entry, startMs, endMs } of spanned) {
|
|
215
|
-
const slice = inferSearchType(entry);
|
|
216
|
-
let covered = coveredBySearchType.get(slice);
|
|
217
|
-
if (!covered) {
|
|
218
|
-
covered = /* @__PURE__ */ new Set();
|
|
219
|
-
coveredBySearchType.set(slice, covered);
|
|
220
|
-
}
|
|
221
|
-
let fullyCovered = true;
|
|
222
|
-
for (let dayMs = startMs; dayMs <= endMs; dayMs += MS_PER_DAY) if (!covered.has(dayMs)) {
|
|
223
|
-
fullyCovered = false;
|
|
224
|
-
break;
|
|
225
|
-
}
|
|
226
|
-
if (fullyCovered) {
|
|
227
|
-
subsumed.push(entry);
|
|
228
|
-
continue;
|
|
229
|
-
}
|
|
230
|
-
kept.push(entry);
|
|
231
|
-
for (let dayMs = startMs; dayMs <= endMs; dayMs += MS_PER_DAY) covered.add(dayMs);
|
|
232
|
-
}
|
|
233
|
-
return {
|
|
234
|
-
kept,
|
|
235
|
-
subsumed
|
|
236
|
-
};
|
|
237
|
-
}
|
|
238
|
-
function dedupeOverlappingTiers(entries, queryRange) {
|
|
239
|
-
return splitOverlappingTiers(entries, queryRange).kept;
|
|
240
|
-
}
|
|
241
|
-
function monthEndMs(month) {
|
|
242
|
-
const [y, m] = month.split("-").map(Number);
|
|
243
|
-
return Date.UTC(y, m, 0, 23, 59, 59, 999);
|
|
244
|
-
}
|
|
245
|
-
function quarterEndMs(quarter) {
|
|
246
|
-
const [yStr, qStr] = quarter.split("-Q");
|
|
247
|
-
const y = Number(yStr);
|
|
248
|
-
const q = Number(qStr);
|
|
249
|
-
return Date.UTC(y, q * 3, 0, 23, 59, 59, 999);
|
|
250
|
-
}
|
|
251
|
-
export { compactTieredImpl, countRawDailies, dedupeOverlappingTiers, enumeratePartitions, splitOverlappingTiers };
|
|
@@ -1,82 +0,0 @@
|
|
|
1
|
-
import { ManifestStore, TableName } from "../_chunks/storage.mjs";
|
|
2
|
-
interface R2ObjectMetadata {
|
|
3
|
-
etag: string;
|
|
4
|
-
}
|
|
5
|
-
interface R2ObjectBody extends R2ObjectMetadata {
|
|
6
|
-
text: () => Promise<string>;
|
|
7
|
-
}
|
|
8
|
-
interface R2ListResult {
|
|
9
|
-
objects: Array<{
|
|
10
|
-
key: string;
|
|
11
|
-
}>;
|
|
12
|
-
truncated: boolean;
|
|
13
|
-
cursor?: string;
|
|
14
|
-
}
|
|
15
|
-
interface R2ConditionalPutOptions {
|
|
16
|
-
/**
|
|
17
|
-
* Workers-binding-style precondition. `etagMatches` rejects with `null`
|
|
18
|
-
* return on mismatch; `etagDoesNotMatch: '*'` rejects if the key exists.
|
|
19
|
-
*/
|
|
20
|
-
onlyIf?: {
|
|
21
|
-
etagMatches?: string;
|
|
22
|
-
etagDoesNotMatch?: string;
|
|
23
|
-
};
|
|
24
|
-
}
|
|
25
|
-
/**
|
|
26
|
-
* Minimal Cloudflare R2 binding shape needed for the manifest CAS loop.
|
|
27
|
-
* Structurally compatible with Cloudflare's `R2Bucket` Workers API.
|
|
28
|
-
*/
|
|
29
|
-
interface R2ManifestBucketLike {
|
|
30
|
-
get: (key: string) => Promise<R2ObjectBody | null>;
|
|
31
|
-
put: (key: string, bytes: string | Uint8Array, options?: R2ConditionalPutOptions) => Promise<R2ObjectMetadata | null>;
|
|
32
|
-
list: (options?: {
|
|
33
|
-
prefix?: string;
|
|
34
|
-
cursor?: string;
|
|
35
|
-
limit?: number;
|
|
36
|
-
}) => Promise<R2ListResult>;
|
|
37
|
-
/**
|
|
38
|
-
* Bulk delete. Required by {@link ManifestStore.purgeTenant}. Cloudflare's
|
|
39
|
-
* `R2Bucket.delete` accepts a single key or a string[] batch; both shapes
|
|
40
|
-
* work here.
|
|
41
|
-
*/
|
|
42
|
-
delete: (keys: string | string[]) => Promise<void>;
|
|
43
|
-
}
|
|
44
|
-
/**
|
|
45
|
-
* CAS lifecycle events emitted by the manifest store. Consumers wire these
|
|
46
|
-
* into metrics (prom-client, console.table, the contention harness) to
|
|
47
|
-
* measure rejection rate and latency under real R2 load.
|
|
48
|
-
*/
|
|
49
|
-
type R2ManifestEvent = {
|
|
50
|
-
kind: 'cas-attempt';
|
|
51
|
-
siteId: string;
|
|
52
|
-
table: TableName;
|
|
53
|
-
attempt: number;
|
|
54
|
-
} | {
|
|
55
|
-
kind: 'cas-rejected';
|
|
56
|
-
siteId: string;
|
|
57
|
-
table: TableName;
|
|
58
|
-
attempt: number;
|
|
59
|
-
} | {
|
|
60
|
-
kind: 'cas-committed';
|
|
61
|
-
siteId: string;
|
|
62
|
-
table: TableName;
|
|
63
|
-
attempts: number;
|
|
64
|
-
};
|
|
65
|
-
interface CreateR2ManifestStoreOptions {
|
|
66
|
-
bucket: R2ManifestBucketLike;
|
|
67
|
-
/** Tenant scope. All shard keys are prefixed `u_<userId>/manifest/...`. */
|
|
68
|
-
userId: string;
|
|
69
|
-
/** Override the snapshot version-id generator. Defaults to `${ts}-${random}`. */
|
|
70
|
-
newSnapshotId?: () => string;
|
|
71
|
-
now?: () => number;
|
|
72
|
-
/** Maximum CAS retries before giving up. Defaults to 8. */
|
|
73
|
-
maxRetries?: number;
|
|
74
|
-
/**
|
|
75
|
-
* Optional telemetry hook. Fired synchronously from the CAS loop on each
|
|
76
|
-
* attempt, rejection, and successful commit. Must not throw; exceptions
|
|
77
|
-
* propagate and will fail the mutation.
|
|
78
|
-
*/
|
|
79
|
-
onEvent?: (event: R2ManifestEvent) => void;
|
|
80
|
-
}
|
|
81
|
-
declare function createR2ManifestStore(opts: CreateR2ManifestStoreOptions): ManifestStore;
|
|
82
|
-
export { CreateR2ManifestStoreOptions, R2ManifestBucketLike, R2ManifestEvent, createR2ManifestStore };
|