@gscdump/engine 0.40.2 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +4 -4
  2. package/dist/_chunks/duckdb.d.mts +2 -9
  3. package/dist/_chunks/engine.mjs +3 -7
  4. package/dist/_chunks/entities.mjs +1 -1
  5. package/dist/_chunks/index.d.mts +3 -8
  6. package/dist/_chunks/index2.d.mts +1 -2
  7. package/dist/_chunks/libs/hyparquet-compressors.mjs +1 -1
  8. package/dist/_chunks/libs/hyparquet.mjs +1 -63
  9. package/dist/_chunks/libs/icebird.d.mts +1 -81
  10. package/dist/_chunks/libs/icebird.mjs +4 -330
  11. package/dist/_chunks/parquet-plan.mjs +248 -3
  12. package/dist/_chunks/resolver.mjs +16 -2
  13. package/dist/_chunks/schema.d.mts +2 -2
  14. package/dist/_chunks/sink.d.mts +4 -5
  15. package/dist/_chunks/source.mjs +3 -3
  16. package/dist/_chunks/storage.d.mts +1 -43
  17. package/dist/_chunks/types.d.mts +8 -0
  18. package/dist/adapters/node.d.mts +1 -11
  19. package/dist/adapters/node.mjs +1 -1
  20. package/dist/analyzer/index.d.mts +1 -1
  21. package/dist/entities.d.mts +2 -10
  22. package/dist/entities.mjs +2 -2
  23. package/dist/iceberg/index.d.mts +2 -3
  24. package/dist/iceberg/index.mjs +12 -12
  25. package/dist/index.d.mts +83 -2
  26. package/dist/index.mjs +365 -6
  27. package/dist/period/index.d.mts +2 -2
  28. package/dist/period/index.mjs +1 -1
  29. package/dist/planner.mjs +1 -2
  30. package/dist/resolver/index.d.mts +2 -9
  31. package/dist/resolver/index.mjs +2 -2
  32. package/dist/rollups.d.mts +3 -20
  33. package/dist/rollups.mjs +5 -5
  34. package/dist/scope.d.mts +1 -2
  35. package/dist/scope.mjs +1 -1
  36. package/dist/source/index.d.mts +2 -2
  37. package/dist/source/index.mjs +2 -2
  38. package/dist/sql-bind.d.mts +1 -29
  39. package/dist/sql-bind.mjs +1 -1
  40. package/package.json +7 -22
  41. package/dist/_chunks/compaction.mjs +0 -251
  42. package/dist/adapters/r2-manifest.d.mts +0 -82
  43. package/dist/adapters/r2-manifest.mjs +0 -364
  44. package/dist/compaction-public.d.mts +0 -15
  45. package/dist/compaction-public.mjs +0 -5
  46. package/dist/snapshot.d.mts +0 -2
  47. package/dist/snapshot.mjs +0 -1
package/dist/rollups.mjs CHANGED
@@ -1,9 +1,9 @@
1
- import "./_chunks/layout.mjs";
2
1
  import { engineErrors } from "./errors.mjs";
2
+ import "./_chunks/layout.mjs";
3
3
  import { encodeRowsToParquetFlex } from "./adapters/hyparquet.mjs";
4
4
  import { createIndexingMetadataStore, createQueryDimStore, createSitemapStore, inspectionParquetKey, sitemapUrlsIndexPrefix } from "./_chunks/entities.mjs";
5
- import { encodeJsonBigintSafe } from "@gscdump/lakehouse";
6
5
  import { MS_PER_DAY } from "gscdump/dates";
6
+ import { encodeJsonBigintSafe } from "@gscdump/lakehouse";
7
7
  function rollupPrefix(ctx, searchType) {
8
8
  const base = ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/rollups` : `u_${ctx.userId}/rollups`;
9
9
  return searchType !== void 0 && searchType !== "web" ? `${base}/${searchType}` : base;
@@ -754,7 +754,7 @@ const queryCanonicalDailyRollup = {
754
754
  orderBy: "date, query_canonical",
755
755
  pageRows: ROLLUP_PAGE_ROWS_DAILY
756
756
  },
757
- maxWindowDays: 7,
757
+ maxWindowDays: DAILY_MAX_WINDOW_DAYS,
758
758
  sqlFor: dailyWindowSqlFor(useDim, canonExpr)
759
759
  })).map(mapDailyRow);
760
760
  }
@@ -800,7 +800,7 @@ async function rebuildCanonicalDailyResumable(opts) {
800
800
  const sType = searchType !== void 0 ? { searchType } : {};
801
801
  const startPageOffset = opts.pageOffset ?? 0;
802
802
  const pageRows = opts.pageRows ?? 7e4;
803
- const maxWindowDays = opts.maxWindowDays ?? 7;
803
+ const maxWindowDays = opts.maxWindowDays ?? DAILY_MAX_WINDOW_DAYS;
804
804
  const shardCount = Math.max(1, Math.floor(opts.shardCount ?? 1));
805
805
  const windows = planRollupWindows((await engine.listPartitions({
806
806
  ctx,
@@ -1261,4 +1261,4 @@ const DEFAULT_ROLLUPS = [
1261
1261
  sitemapChanges28dRollup
1262
1262
  ];
1263
1263
  const CANONICAL_ROLLUPS = [queryCanonicalVariantsRollup, queryCanonicalDailyRollup];
1264
- export { CANONICAL_ROLLUPS, DAILY_MAX_WINDOW_DAYS, DEFAULT_ROLLUPS, ROLLUP_PAGE_ROWS, ROLLUP_PAGE_ROWS_DAILY, ROLLUP_PAGE_ROWS_WIDE, WINDOW_BYTE_BUDGET, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, partitionDaySpan, partitionsInRange, planRollupWindows, queryCanonicalDailyRollup, queryCanonicalVariantsRollup, readLatestRollup, rebuildCanonicalDailyResumable, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, runWindowed, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };
1264
+ export { CANONICAL_ROLLUPS, DEFAULT_ROLLUPS, ROLLUP_PAGE_ROWS, ROLLUP_PAGE_ROWS_DAILY, ROLLUP_PAGE_ROWS_WIDE, WINDOW_BYTE_BUDGET, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, planRollupWindows, queryCanonicalDailyRollup, queryCanonicalVariantsRollup, readLatestRollup, rebuildCanonicalDailyResumable, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, runWindowed, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topPages28dRollup, weeklyTotalsRollup };
package/dist/scope.d.mts CHANGED
@@ -28,7 +28,6 @@ interface TableScope {
28
28
  window?: ResolvedWindow;
29
29
  siteId?: string;
30
30
  }
31
- declare function buildTableScope(table: Record<string, any>, opts: ScopedRunnerOptions): TableScope;
32
31
  declare function mergeScope(scope: TableScope, ...extra: SQL[]): SQL | undefined;
33
32
  /**
34
33
  * Bind `buildTableScope` + `mergeScope` to a specific drizzle schema. Engine
@@ -41,4 +40,4 @@ declare function createScopedHelpers<S extends Record<string, Record<string, any
41
40
  scopeFor: (table: keyof S, opts: ScopedRunnerOptions) => TableScope;
42
41
  mergeScope: typeof mergeScope;
43
42
  };
44
- export { ResolvedWindow, ScopedRunnerOptions, TableScope, buildTableScope, createScopedHelpers, mergeScope };
43
+ export { ResolvedWindow, ScopedRunnerOptions, TableScope, createScopedHelpers, mergeScope };
package/dist/scope.mjs CHANGED
@@ -27,4 +27,4 @@ function createScopedHelpers(schema) {
27
27
  mergeScope
28
28
  };
29
29
  }
30
- export { buildTableScope, createScopedHelpers, mergeScope };
30
+ export { createScopedHelpers, mergeScope };
@@ -1,3 +1,3 @@
1
1
  import { AnalysisQuerySource, AnalysisSourceKind, ExecuteSqlOptions, FileSet, QueryRow, SourceCapabilities } from "../_chunks/registry.mjs";
2
- import { AttachedTableMissingError, AttachedTableRunner, AttachedTableSourceOptions, CreateSqlQuerySourceOptions, ENGINE_QUERY_CAPABILITIES, EngineQuerySourceOptions, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows, rewriteForTableSource, runAnalyzerWithEngine } from "../_chunks/index.mjs";
3
- export { type AnalysisQuerySource, type AnalysisSourceKind, AttachedTableMissingError, type AttachedTableRunner, type AttachedTableSourceOptions, type CreateSqlQuerySourceOptions, ENGINE_QUERY_CAPABILITIES, EngineQuerySourceOptions, type ExecuteSqlOptions, type FileSet, type QueryRow, type SourceCapabilities, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows, rewriteForTableSource, runAnalyzerWithEngine };
2
+ import { AttachedTableMissingError, AttachedTableRunner, AttachedTableSourceOptions, CreateSqlQuerySourceOptions, ENGINE_QUERY_CAPABILITIES, EngineQuerySourceOptions, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows, runAnalyzerWithEngine } from "../_chunks/index.mjs";
3
+ export { type AnalysisQuerySource, type AnalysisSourceKind, AttachedTableMissingError, type AttachedTableRunner, type AttachedTableSourceOptions, type CreateSqlQuerySourceOptions, ENGINE_QUERY_CAPABILITIES, EngineQuerySourceOptions, type ExecuteSqlOptions, type FileSet, type QueryRow, type SourceCapabilities, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows, runAnalyzerWithEngine };
@@ -1,2 +1,2 @@
1
- import { AttachedTableMissingError, ENGINE_QUERY_CAPABILITIES, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows, rewriteForTableSource, runAnalyzerWithEngine } from "../_chunks/source.mjs";
2
- export { AttachedTableMissingError, ENGINE_QUERY_CAPABILITIES, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows, rewriteForTableSource, runAnalyzerWithEngine };
1
+ import { AttachedTableMissingError, ENGINE_QUERY_CAPABILITIES, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows, runAnalyzerWithEngine } from "../_chunks/source.mjs";
2
+ export { AttachedTableMissingError, ENGINE_QUERY_CAPABILITIES, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows, runAnalyzerWithEngine };
@@ -1,33 +1,5 @@
1
- import { EngineError } from "./_chunks/errors.mjs";
2
- import { Result } from "gscdump/result";
3
1
  /** Escape single quotes for inlining inside a SQL string literal (SQL-standard `''` escaping). */
4
2
  declare function sqlEscape(s: string): string;
5
- /**
6
- * Errors-as-values core for {@link formatLiteral}: returns a typed
7
- * `invalid-sql-literal` `EngineError` for values that can't be safely inlined.
8
- */
9
- declare function formatLiteralResult(value: unknown): Result<string, EngineError>;
10
3
  declare function formatLiteral(value: unknown): string;
11
- /**
12
- * Replace `?` and `$N` placeholders with inline SQL literals. Single-quoted
13
- * string regions and SQL comments (`-- line`, `/* block *\/`) are left
14
- * untouched — a `?` or `$1` inside `'foo?bar'` or a comment is not a
15
- * placeholder. SQL-standard `''` escape handling; no `\`-escape or
16
- * dialect-specific identifier quoting.
17
- *
18
- * `?` placeholders bind sequentially against `params`. `$N` (Postgres-style)
19
- * binds explicitly to `params[N-1]`. The two styles must not be mixed in the
20
- * same query.
21
- *
22
- * Throws when placeholder count and params length disagree, or when a `$N`
23
- * index is out of range.
24
- */
25
- /**
26
- * Errors-as-values core for {@link bindLiterals}: returns a typed
27
- * `placeholder-arity-mismatch` / `invalid-sql-literal` `EngineError` instead of
28
- * throwing, so the edge RPC / proxy callers that build SQL from untrusted params
29
- * can branch on the failure.
30
- */
31
- declare function bindLiteralsResult(sql: string, params: readonly unknown[]): Result<string, EngineError>;
32
4
  declare function bindLiterals(sql: string, params: readonly unknown[]): string;
33
- export { bindLiterals, bindLiteralsResult, formatLiteral, formatLiteralResult, sqlEscape };
5
+ export { bindLiterals, formatLiteral, sqlEscape };
package/dist/sql-bind.mjs CHANGED
@@ -101,4 +101,4 @@ function bindLiteralsResult(sql, params) {
101
101
  function bindLiterals(sql, params) {
102
102
  return unwrapResult(bindLiteralsResult(sql, params), engineErrorToException);
103
103
  }
104
- export { bindLiterals, bindLiteralsResult, formatLiteral, formatLiteralResult, sqlEscape };
104
+ export { bindLiterals, formatLiteral, sqlEscape };
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@gscdump/engine",
3
3
  "type": "module",
4
- "version": "0.40.2",
4
+ "version": "1.0.1",
5
5
  "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
6
6
  "author": {
7
7
  "name": "Harlan Wilton",
@@ -36,21 +36,11 @@
36
36
  "import": "./dist/errors.mjs",
37
37
  "default": "./dist/errors.mjs"
38
38
  },
39
- "./snapshot": {
40
- "types": "./dist/snapshot.d.mts",
41
- "import": "./dist/snapshot.mjs",
42
- "default": "./dist/snapshot.mjs"
43
- },
44
39
  "./planner": {
45
40
  "types": "./dist/planner.d.mts",
46
41
  "import": "./dist/planner.mjs",
47
42
  "default": "./dist/planner.mjs"
48
43
  },
49
- "./compaction": {
50
- "types": "./dist/compaction-public.d.mts",
51
- "import": "./dist/compaction-public.mjs",
52
- "default": "./dist/compaction-public.mjs"
53
- },
54
44
  "./schema": {
55
45
  "types": "./dist/schema.d.mts",
56
46
  "import": "./dist/schema.mjs",
@@ -111,11 +101,6 @@
111
101
  "import": "./dist/adapters/r2.mjs",
112
102
  "default": "./dist/adapters/r2.mjs"
113
103
  },
114
- "./r2-manifest": {
115
- "types": "./dist/adapters/r2-manifest.d.mts",
116
- "import": "./dist/adapters/r2-manifest.mjs",
117
- "default": "./dist/adapters/r2-manifest.mjs"
118
- },
119
104
  "./resolver": {
120
105
  "types": "./dist/resolver/index.d.mts",
121
106
  "import": "./dist/resolver/index.mjs",
@@ -168,10 +153,10 @@
168
153
  "dist"
169
154
  ],
170
155
  "engines": {
171
- "node": ">=18"
156
+ "node": ">=22"
172
157
  },
173
158
  "peerDependencies": {
174
- "@duckdb/duckdb-wasm": "^1.32.0"
159
+ "@duckdb/duckdb-wasm": "^1.33.1-dev57.0"
175
160
  },
176
161
  "peerDependenciesMeta": {
177
162
  "@duckdb/duckdb-wasm": {
@@ -181,12 +166,12 @@
181
166
  "dependencies": {
182
167
  "drizzle-orm": "1.0.0-rc.3",
183
168
  "proper-lockfile": "^4.1.2",
184
- "@gscdump/contracts": "0.40.2",
185
- "@gscdump/lakehouse": "0.40.2",
186
- "gscdump": "0.40.2"
169
+ "@gscdump/contracts": "^1.0.1",
170
+ "gscdump": "^1.0.1",
171
+ "@gscdump/lakehouse": "^1.0.1"
187
172
  },
188
173
  "devDependencies": {
189
- "@duckdb/duckdb-wasm": "^1.32.0",
174
+ "@duckdb/duckdb-wasm": "1.33.1-dev57.0",
190
175
  "@types/node": "^26.1.1",
191
176
  "@types/proper-lockfile": "^4.1.4",
192
177
  "aws4fetch": "^1.0.20",
@@ -1,251 +0,0 @@
1
- import { dayPartition, inferSearchType, mondayOfWeek, monthPartition, objectKey, quarterOfMonth, quarterPartition, weekPartition } from "./layout.mjs";
2
- import { currentSchemaVersion } from "./schema.mjs";
3
- import { MS_PER_DAY } from "gscdump/dates";
4
- const DAILY_PARTITION_RE = /^daily\/(\d{4}-\d{2}-\d{2})$/;
5
- const WEEKLY_PARTITION_RE = /^weekly\/(\d{4}-\d{2}-\d{2})$/;
6
- const MONTHLY_PARTITION_RE = /^monthly\/(\d{4}-\d{2})$/;
7
- const QUARTERLY_PARTITION_RE = /^quarterly\/(\d{4})-Q([1-4])$/;
8
- const DEFAULT_THRESHOLDS = {
9
- raw: 7,
10
- d7: 30,
11
- d30: 90
12
- };
13
- function countRawDailies(entries) {
14
- return entries.filter((e) => e.tier === "raw" || e.tier == null && e.partition.startsWith("daily/")).length;
15
- }
16
- const PENDING_WINDOW_DAYS = 4;
17
- const STAGES = [
18
- {
19
- inputTier: "raw",
20
- outputTier: "d7",
21
- cutoffDays: DEFAULT_THRESHOLDS.raw,
22
- bucketKey: (e) => {
23
- const m = e.partition.match(DAILY_PARTITION_RE);
24
- if (!m) return void 0;
25
- return mondayOfWeek(m[1]);
26
- },
27
- bucketLatestMs: (monday) => Date.parse(`${monday}T00:00:00Z`) + 6 * MS_PER_DAY,
28
- outputPartition: weekPartition
29
- },
30
- {
31
- inputTier: "d7",
32
- outputTier: "d30",
33
- cutoffDays: DEFAULT_THRESHOLDS.d7,
34
- bucketKey: (e) => {
35
- const m = e.partition.match(WEEKLY_PARTITION_RE);
36
- if (!m) return void 0;
37
- return m[1].slice(0, 7);
38
- },
39
- bucketLatestMs: monthEndMs,
40
- outputPartition: monthPartition
41
- },
42
- {
43
- inputTier: "d30",
44
- outputTier: "d90",
45
- cutoffDays: DEFAULT_THRESHOLDS.d30,
46
- bucketKey: (e) => {
47
- const m = e.partition.match(MONTHLY_PARTITION_RE);
48
- if (!m) return void 0;
49
- return quarterOfMonth(m[1]);
50
- },
51
- bucketLatestMs: quarterEndMs,
52
- outputPartition: quarterPartition
53
- }
54
- ];
55
- async function compactTieredImpl(deps, ctx, now, overrides = {}) {
56
- const thresholds = {
57
- ...DEFAULT_THRESHOLDS,
58
- ...overrides
59
- };
60
- const stagesWithThresholds = STAGES.map((s) => ({
61
- ...s,
62
- cutoffDays: s.outputTier === "d7" ? thresholds.raw : s.outputTier === "d30" ? thresholds.d7 : thresholds.d30
63
- }));
64
- for (const stage of stagesWithThresholds) await runStage(deps, ctx, stage, now);
65
- }
66
- async function runStage(deps, ctx, stage, now) {
67
- const cutoff = now - Math.max(stage.cutoffDays, PENDING_WINDOW_DAYS) * MS_PER_DAY;
68
- const candidates = await deps.manifestStore.listLive({
69
- userId: ctx.userId,
70
- siteId: ctx.siteId,
71
- table: ctx.table,
72
- tier: stage.inputTier
73
- });
74
- const buckets = /* @__PURE__ */ new Map();
75
- for (const entry of candidates) {
76
- if (entry.partition.startsWith("hourly/")) continue;
77
- const key = stage.bucketKey(entry);
78
- if (!key) continue;
79
- if (stage.bucketLatestMs(key) >= cutoff) continue;
80
- const compositeKey = `${inferSearchType(entry)}\0${key}`;
81
- if (!buckets.has(compositeKey)) buckets.set(compositeKey, []);
82
- buckets.get(compositeKey).push(entry);
83
- }
84
- for (const [compositeKey, entries] of buckets) {
85
- const [searchType, bucket] = compositeKey.split("\0");
86
- const targetPartition = stage.outputPartition(bucket);
87
- if (entries.length === 1 && entries[0].partition === targetPartition) continue;
88
- await deps.manifestStore.withLock({
89
- userId: ctx.userId,
90
- siteId: ctx.siteId,
91
- table: ctx.table,
92
- partition: targetPartition
93
- }, async () => {
94
- const key = objectKey(ctx, ctx.table, targetPartition, now, searchType);
95
- const { bytes, rowCount } = await deps.codec.compactRows({ table: ctx.table }, entries.map((e) => e.objectKey), key, deps.dataSource);
96
- const newEntry = {
97
- userId: ctx.userId,
98
- siteId: ctx.siteId,
99
- table: ctx.table,
100
- partition: targetPartition,
101
- objectKey: key,
102
- rowCount,
103
- bytes,
104
- createdAt: now,
105
- schemaVersion: currentSchemaVersion(ctx.table),
106
- tier: stage.outputTier,
107
- ...searchType !== "web" ? { searchType } : {}
108
- };
109
- await deps.manifestStore.registerVersion(newEntry, entries);
110
- });
111
- }
112
- }
113
- function enumeratePartitions(startDate, endDate) {
114
- const out = [];
115
- const [sy, sm, sd] = startDate.split("-").map(Number);
116
- const [ey, em, ed] = endDate.split("-").map(Number);
117
- const start = Date.UTC(sy, sm - 1, sd);
118
- const end = Date.UTC(ey, em - 1, ed);
119
- if (end < start) return out;
120
- const seenWeeks = /* @__PURE__ */ new Set();
121
- const seenMonths = /* @__PURE__ */ new Set();
122
- const seenQuarters = /* @__PURE__ */ new Set();
123
- for (let t = start; t <= end; t += 864e5) {
124
- const d = new Date(t);
125
- const y = d.getUTCFullYear();
126
- const m = String(d.getUTCMonth() + 1).padStart(2, "0");
127
- const isoDay = `${y}-${m}-${String(d.getUTCDate()).padStart(2, "0")}`;
128
- const isoMonth = `${y}-${m}`;
129
- out.push(dayPartition(isoDay));
130
- const monday = mondayOfWeek(isoDay);
131
- if (!seenWeeks.has(monday)) {
132
- seenWeeks.add(monday);
133
- out.push(weekPartition(monday));
134
- }
135
- if (!seenMonths.has(isoMonth)) {
136
- seenMonths.add(isoMonth);
137
- out.push(monthPartition(isoMonth));
138
- }
139
- const quarter = quarterOfMonth(isoMonth);
140
- if (!seenQuarters.has(quarter)) {
141
- seenQuarters.add(quarter);
142
- out.push(quarterPartition(quarter));
143
- }
144
- }
145
- return out;
146
- }
147
- function partitionSpan(partition) {
148
- let m = partition.match(DAILY_PARTITION_RE);
149
- if (m) {
150
- const ms = Date.parse(`${m[1]}T00:00:00Z`);
151
- return {
152
- rank: 0,
153
- startMs: ms,
154
- endMs: ms
155
- };
156
- }
157
- m = partition.match(WEEKLY_PARTITION_RE);
158
- if (m) {
159
- const ms = Date.parse(`${m[1]}T00:00:00Z`);
160
- return {
161
- rank: 1,
162
- startMs: ms,
163
- endMs: ms + 6 * MS_PER_DAY
164
- };
165
- }
166
- m = partition.match(MONTHLY_PARTITION_RE);
167
- if (m) {
168
- const [y, mo] = m[1].split("-").map(Number);
169
- return {
170
- rank: 2,
171
- startMs: Date.UTC(y, mo - 1, 1),
172
- endMs: Date.UTC(y, mo, 0)
173
- };
174
- }
175
- m = partition.match(QUARTERLY_PARTITION_RE);
176
- if (m) {
177
- const y = Number(m[1]);
178
- const q = Number(m[2]);
179
- return {
180
- rank: 3,
181
- startMs: Date.UTC(y, (q - 1) * 3, 1),
182
- endMs: Date.UTC(y, q * 3, 0)
183
- };
184
- }
185
- }
186
- function splitOverlappingTiers(entries, queryRange) {
187
- const rangeStartMs = queryRange ? Date.parse(`${queryRange.start}T00:00:00Z`) : void 0;
188
- const rangeEndMs = queryRange ? Date.parse(`${queryRange.end}T00:00:00Z`) : void 0;
189
- const spanned = [];
190
- const kept = [];
191
- const subsumed = [];
192
- const clampToRange = Number.isFinite(rangeStartMs) && Number.isFinite(rangeEndMs);
193
- for (const entry of entries) {
194
- const span = partitionSpan(entry.partition);
195
- if (!span) {
196
- kept.push(entry);
197
- continue;
198
- }
199
- const startMs = clampToRange ? Math.max(span.startMs, rangeStartMs) : span.startMs;
200
- const endMs = clampToRange ? Math.min(span.endMs, rangeEndMs) : span.endMs;
201
- if (queryRange && startMs > endMs) {
202
- subsumed.push(entry);
203
- continue;
204
- }
205
- spanned.push({
206
- entry,
207
- rank: span.rank,
208
- startMs,
209
- endMs
210
- });
211
- }
212
- spanned.sort((a, b) => a.rank - b.rank || b.entry.createdAt - a.entry.createdAt);
213
- const coveredBySearchType = /* @__PURE__ */ new Map();
214
- for (const { entry, startMs, endMs } of spanned) {
215
- const slice = inferSearchType(entry);
216
- let covered = coveredBySearchType.get(slice);
217
- if (!covered) {
218
- covered = /* @__PURE__ */ new Set();
219
- coveredBySearchType.set(slice, covered);
220
- }
221
- let fullyCovered = true;
222
- for (let dayMs = startMs; dayMs <= endMs; dayMs += MS_PER_DAY) if (!covered.has(dayMs)) {
223
- fullyCovered = false;
224
- break;
225
- }
226
- if (fullyCovered) {
227
- subsumed.push(entry);
228
- continue;
229
- }
230
- kept.push(entry);
231
- for (let dayMs = startMs; dayMs <= endMs; dayMs += MS_PER_DAY) covered.add(dayMs);
232
- }
233
- return {
234
- kept,
235
- subsumed
236
- };
237
- }
238
- function dedupeOverlappingTiers(entries, queryRange) {
239
- return splitOverlappingTiers(entries, queryRange).kept;
240
- }
241
- function monthEndMs(month) {
242
- const [y, m] = month.split("-").map(Number);
243
- return Date.UTC(y, m, 0, 23, 59, 59, 999);
244
- }
245
- function quarterEndMs(quarter) {
246
- const [yStr, qStr] = quarter.split("-Q");
247
- const y = Number(yStr);
248
- const q = Number(qStr);
249
- return Date.UTC(y, q * 3, 0, 23, 59, 59, 999);
250
- }
251
- export { compactTieredImpl, countRawDailies, dedupeOverlappingTiers, enumeratePartitions, splitOverlappingTiers };
@@ -1,82 +0,0 @@
1
- import { ManifestStore, TableName } from "../_chunks/storage.mjs";
2
- interface R2ObjectMetadata {
3
- etag: string;
4
- }
5
- interface R2ObjectBody extends R2ObjectMetadata {
6
- text: () => Promise<string>;
7
- }
8
- interface R2ListResult {
9
- objects: Array<{
10
- key: string;
11
- }>;
12
- truncated: boolean;
13
- cursor?: string;
14
- }
15
- interface R2ConditionalPutOptions {
16
- /**
17
- * Workers-binding-style precondition. `etagMatches` rejects with `null`
18
- * return on mismatch; `etagDoesNotMatch: '*'` rejects if the key exists.
19
- */
20
- onlyIf?: {
21
- etagMatches?: string;
22
- etagDoesNotMatch?: string;
23
- };
24
- }
25
- /**
26
- * Minimal Cloudflare R2 binding shape needed for the manifest CAS loop.
27
- * Structurally compatible with Cloudflare's `R2Bucket` Workers API.
28
- */
29
- interface R2ManifestBucketLike {
30
- get: (key: string) => Promise<R2ObjectBody | null>;
31
- put: (key: string, bytes: string | Uint8Array, options?: R2ConditionalPutOptions) => Promise<R2ObjectMetadata | null>;
32
- list: (options?: {
33
- prefix?: string;
34
- cursor?: string;
35
- limit?: number;
36
- }) => Promise<R2ListResult>;
37
- /**
38
- * Bulk delete. Required by {@link ManifestStore.purgeTenant}. Cloudflare's
39
- * `R2Bucket.delete` accepts a single key or a string[] batch; both shapes
40
- * work here.
41
- */
42
- delete: (keys: string | string[]) => Promise<void>;
43
- }
44
- /**
45
- * CAS lifecycle events emitted by the manifest store. Consumers wire these
46
- * into metrics (prom-client, console.table, the contention harness) to
47
- * measure rejection rate and latency under real R2 load.
48
- */
49
- type R2ManifestEvent = {
50
- kind: 'cas-attempt';
51
- siteId: string;
52
- table: TableName;
53
- attempt: number;
54
- } | {
55
- kind: 'cas-rejected';
56
- siteId: string;
57
- table: TableName;
58
- attempt: number;
59
- } | {
60
- kind: 'cas-committed';
61
- siteId: string;
62
- table: TableName;
63
- attempts: number;
64
- };
65
- interface CreateR2ManifestStoreOptions {
66
- bucket: R2ManifestBucketLike;
67
- /** Tenant scope. All shard keys are prefixed `u_<userId>/manifest/...`. */
68
- userId: string;
69
- /** Override the snapshot version-id generator. Defaults to `${ts}-${random}`. */
70
- newSnapshotId?: () => string;
71
- now?: () => number;
72
- /** Maximum CAS retries before giving up. Defaults to 8. */
73
- maxRetries?: number;
74
- /**
75
- * Optional telemetry hook. Fired synchronously from the CAS loop on each
76
- * attempt, rejection, and successful commit. Must not throw; exceptions
77
- * propagate and will fail the mutation.
78
- */
79
- onEvent?: (event: R2ManifestEvent) => void;
80
- }
81
- declare function createR2ManifestStore(opts: CreateR2ManifestStoreOptions): ManifestStore;
82
- export { CreateR2ManifestStoreOptions, R2ManifestBucketLike, R2ManifestEvent, createR2ManifestStore };