@gscdump/engine 3.0.0 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/coerce.d.mts +1 -2
- package/dist/engine.d.mts +1 -2
- package/dist/index.d.mts +5 -5
- package/dist/index.mjs +5 -5
- package/dist/layout.d.mts +1 -9
- package/dist/period/index.d.mts +2 -20
- package/dist/period/index.mjs +1 -75
- package/dist/profile.d.mts +1 -1
- package/package.json +9 -4
package/dist/coerce.d.mts
CHANGED
package/dist/engine.d.mts
CHANGED
package/dist/index.d.mts
CHANGED
|
@@ -5,11 +5,11 @@ import { SnapshotIndex } from "./snapshot.mjs";
|
|
|
5
5
|
import { InspectionVerdict, SchedulePolicy, ScheduleState, fixedPolicy, inspectionPolicy, sitemapPolicy } from "./schedule.mjs";
|
|
6
6
|
import { Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult } from "./sink.mjs";
|
|
7
7
|
import { CreateR2ManifestStoreOptions, R2ManifestBucketLike, R2ManifestEvent, createR2ManifestStore } from "./adapters/r2-manifest.mjs";
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import { DEFAULT_SEARCH_TYPE,
|
|
8
|
+
import { coerceRows } from "./coerce.mjs";
|
|
9
|
+
import { createStorageEngine } from "./engine.mjs";
|
|
10
|
+
import { DEFAULT_SEARCH_TYPE, inferLegacyTier, inferSearchType, objectKey } from "./layout.mjs";
|
|
11
11
|
import { collectSpans, createQueryProfiler } from "./profile.mjs";
|
|
12
12
|
import { InMemorySink, StoredRow, createInMemorySink } from "./sinks/in-memory-sink.mjs";
|
|
13
13
|
import "./sinks/index.mjs";
|
|
14
|
-
import { DateWeight, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, ROW_LIMIT_R2, SyncTableName, TABLES_BY_SEARCH_TYPE, TABLE_TIERS,
|
|
15
|
-
export { type CodecCtx, type CompactionThresholds, type CompactionTier, type CreateR2ManifestStoreOptions, DEFAULT_SEARCH_TYPE, type DataSource, type DateWeight, type DuckDBFactory, type DuckDBHandle, type EngineOptions, type FileSetRef, type GcCtx, type Grain, type InMemorySink, type InspectionVerdict, type ListLiveFilter, type LockScope,
|
|
14
|
+
import { DateWeight, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, ROW_LIMIT_R2, SyncTableName, TABLES_BY_SEARCH_TYPE, TABLE_TIERS, TableTier, TieredTableName, getTableTier, parseEnabledSearchTypes, validateEnabledSearchTypes } from "./sync-config.mjs";
|
|
15
|
+
export { type CodecCtx, type CompactionThresholds, type CompactionTier, type CreateR2ManifestStoreOptions, DEFAULT_SEARCH_TYPE, type DataSource, type DateWeight, type DuckDBFactory, type DuckDBHandle, type EngineOptions, type FileSetRef, type GcCtx, type Grain, type InMemorySink, type InspectionVerdict, type ListLiveFilter, type LockScope, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryProfiler, type QueryResult, type QuerySpan, type R2ManifestBucketLike, type R2ManifestEvent, ROW_LIMIT_R2, type Row, type RunSQLOptions, type SchedulePolicy, type ScheduleState, type SearchType, type Sink, type SinkCapabilities, type SinkCloseResult, type SinkOptions, type SinkSlice, type SinkWriteResult, type SnapshotIndex, type StorageEngine, type StoredRow, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, type SyncTableName, TABLES_BY_SEARCH_TYPE, TABLE_TIERS, type TableName, type TableTier, type TenantCtx, type TieredTableName, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, coerceRows, collectSpans, createDuckDBCodec, createDuckDBExecutor, createInMemorySink, createQueryProfiler, createR2ManifestStore, createStorageEngine, fixedPolicy, getTableTier, inferLegacyTier, inferSearchType, inspectionPolicy, objectKey, parseEnabledSearchTypes, sitemapPolicy, validateEnabledSearchTypes };
|
package/dist/index.mjs
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { DEFAULT_SEARCH_TYPE,
|
|
1
|
+
import { DEFAULT_SEARCH_TYPE, inferLegacyTier, inferSearchType, objectKey } from "./layout.mjs";
|
|
2
2
|
import { createR2ManifestStore } from "./adapters/r2-manifest.mjs";
|
|
3
|
-
import {
|
|
3
|
+
import { coerceRows } from "./coerce.mjs";
|
|
4
4
|
import { createDuckDBCodec, createDuckDBExecutor } from "./duckdb.mjs";
|
|
5
|
-
import {
|
|
5
|
+
import { createStorageEngine } from "./engine.mjs";
|
|
6
6
|
import { collectSpans, createQueryProfiler } from "./profile.mjs";
|
|
7
7
|
import { fixedPolicy, inspectionPolicy, sitemapPolicy } from "./schedule.mjs";
|
|
8
8
|
import { createInMemorySink } from "./sinks/in-memory-sink.mjs";
|
|
9
9
|
import "./sinks/index.mjs";
|
|
10
|
-
import { MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, ROW_LIMIT_R2, TABLES_BY_SEARCH_TYPE, TABLE_TIERS,
|
|
11
|
-
export { DEFAULT_SEARCH_TYPE,
|
|
10
|
+
import { MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, ROW_LIMIT_R2, TABLES_BY_SEARCH_TYPE, TABLE_TIERS, getTableTier, parseEnabledSearchTypes, validateEnabledSearchTypes } from "./sync-config.mjs";
|
|
11
|
+
export { DEFAULT_SEARCH_TYPE, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, ROW_LIMIT_R2, TABLES_BY_SEARCH_TYPE, TABLE_TIERS, coerceRows, collectSpans, createDuckDBCodec, createDuckDBExecutor, createInMemorySink, createQueryProfiler, createR2ManifestStore, createStorageEngine, fixedPolicy, getTableTier, inferLegacyTier, inferSearchType, inspectionPolicy, objectKey, parseEnabledSearchTypes, sitemapPolicy, validateEnabledSearchTypes };
|
package/dist/layout.d.mts
CHANGED
|
@@ -1,14 +1,6 @@
|
|
|
1
1
|
import { CompactionTier, ManifestEntry } from "./storage.mjs";
|
|
2
2
|
import { TableName, TenantCtx } from "@gscdump/contracts";
|
|
3
3
|
import { SearchType } from "gscdump/query";
|
|
4
|
-
declare function dayPartition(date: string): string;
|
|
5
|
-
/**
|
|
6
|
-
* Hourly partition keyed by the PT calendar day (`YYYY-MM-DD`). One parquet
|
|
7
|
-
* per day holds 24 hourly buckets — read-merge-write keeps `(url, hour)`
|
|
8
|
-
* idempotency across retries. Names sort lexically alongside daily ones but
|
|
9
|
-
* never collide because of the `hourly/` prefix.
|
|
10
|
-
*/
|
|
11
|
-
declare function hourPartition(date: string): string;
|
|
12
4
|
/**
|
|
13
5
|
* Default `searchType` for entries written before the field landed and for
|
|
14
6
|
* sync paths that don't request a specific type. GSC's own default; the
|
|
@@ -28,4 +20,4 @@ declare function inferSearchType(entry: Pick<ManifestEntry, 'searchType'>): Sear
|
|
|
28
20
|
* decide how to handle it.
|
|
29
21
|
*/
|
|
30
22
|
declare function inferLegacyTier(entry: Pick<ManifestEntry, 'partition' | 'tier'>): CompactionTier | undefined;
|
|
31
|
-
export { DEFAULT_SEARCH_TYPE,
|
|
23
|
+
export { DEFAULT_SEARCH_TYPE, inferLegacyTier, inferSearchType, objectKey };
|
package/dist/period/index.d.mts
CHANGED
|
@@ -1,22 +1,5 @@
|
|
|
1
1
|
import { AnalysisParams } from "../analysis-types.mjs";
|
|
2
|
-
|
|
3
|
-
type ComparisonMode = 'none' | 'prev-period' | 'yoy';
|
|
4
|
-
interface ResolveWindowOptions {
|
|
5
|
-
preset: WindowPreset;
|
|
6
|
-
comparison?: ComparisonMode;
|
|
7
|
-
anchor?: string;
|
|
8
|
-
start?: string;
|
|
9
|
-
end?: string;
|
|
10
|
-
}
|
|
11
|
-
interface ResolvedWindow {
|
|
12
|
-
start: string;
|
|
13
|
-
end: string;
|
|
14
|
-
days: number;
|
|
15
|
-
comparison?: {
|
|
16
|
-
start: string;
|
|
17
|
-
end: string;
|
|
18
|
-
};
|
|
19
|
-
}
|
|
2
|
+
import { ComparisonMode, ResolveWindowOptions, ResolvedWindow, WindowPreset, resolveWindow } from "gscdump/dates";
|
|
20
3
|
interface AnalysisPeriod {
|
|
21
4
|
startDate: string;
|
|
22
5
|
endDate: string;
|
|
@@ -28,7 +11,6 @@ interface ComparisonPeriod {
|
|
|
28
11
|
declare function defaultEndDate(): string;
|
|
29
12
|
declare function periodOf(params: AnalysisParams): AnalysisPeriod;
|
|
30
13
|
declare function comparisonOf(params: AnalysisParams): ComparisonPeriod;
|
|
31
|
-
declare function resolveWindow(opts: ResolveWindowOptions): ResolvedWindow;
|
|
32
14
|
interface PadTimeseriesOptions<T> {
|
|
33
15
|
/** ISO date (YYYY-MM-DD), inclusive lower bound. */
|
|
34
16
|
startDate: string;
|
|
@@ -50,4 +32,4 @@ type DateRowShape = Record<string, unknown> & {
|
|
|
50
32
|
* once. Existing dates keep all their rows (grouped timeseries safe).
|
|
51
33
|
*/
|
|
52
34
|
declare function padTimeseries<T extends DateRowShape = DateRowShape>(rows: readonly T[], options: PadTimeseriesOptions<T>): T[];
|
|
53
|
-
export { AnalysisPeriod, ComparisonMode, ComparisonPeriod, PadTimeseriesOptions, ResolveWindowOptions, ResolvedWindow, WindowPreset, comparisonOf, defaultEndDate, padTimeseries, periodOf, resolveWindow };
|
|
35
|
+
export { AnalysisPeriod, type ComparisonMode, ComparisonPeriod, PadTimeseriesOptions, type ResolveWindowOptions, type ResolvedWindow, type WindowPreset, comparisonOf, defaultEndDate, padTimeseries, periodOf, resolveWindow };
|
package/dist/period/index.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { MS_PER_DAY, daysAgoUtc, toIsoDate } from "gscdump/dates";
|
|
1
|
+
import { MS_PER_DAY, daysAgoUtc, resolveWindow, toIsoDate } from "gscdump/dates";
|
|
2
2
|
function defaultEndDate() {
|
|
3
3
|
return daysAgoUtc(3);
|
|
4
4
|
}
|
|
@@ -21,80 +21,6 @@ function comparisonOf(params) {
|
|
|
21
21
|
}
|
|
22
22
|
};
|
|
23
23
|
}
|
|
24
|
-
function parseIso(s) {
|
|
25
|
-
return /* @__PURE__ */ new Date(`${s}T00:00:00Z`);
|
|
26
|
-
}
|
|
27
|
-
function addDays(d, n) {
|
|
28
|
-
return new Date(d.getTime() + n * MS_PER_DAY);
|
|
29
|
-
}
|
|
30
|
-
function daysBetween(start, end) {
|
|
31
|
-
return Math.round((parseIso(end).getTime() - parseIso(start).getTime()) / MS_PER_DAY) + 1;
|
|
32
|
-
}
|
|
33
|
-
function resolveWindow(opts) {
|
|
34
|
-
const anchor = opts.anchor ? parseIso(opts.anchor) : /* @__PURE__ */ new Date();
|
|
35
|
-
const anchorIso = toIsoDate(anchor);
|
|
36
|
-
let start;
|
|
37
|
-
let end;
|
|
38
|
-
switch (opts.preset) {
|
|
39
|
-
case "last-7d":
|
|
40
|
-
end = anchorIso;
|
|
41
|
-
start = toIsoDate(addDays(anchor, -6));
|
|
42
|
-
break;
|
|
43
|
-
case "last-28d":
|
|
44
|
-
end = anchorIso;
|
|
45
|
-
start = toIsoDate(addDays(anchor, -27));
|
|
46
|
-
break;
|
|
47
|
-
case "last-30d":
|
|
48
|
-
end = anchorIso;
|
|
49
|
-
start = toIsoDate(addDays(anchor, -29));
|
|
50
|
-
break;
|
|
51
|
-
case "last-90d":
|
|
52
|
-
end = anchorIso;
|
|
53
|
-
start = toIsoDate(addDays(anchor, -89));
|
|
54
|
-
break;
|
|
55
|
-
case "last-180d":
|
|
56
|
-
end = anchorIso;
|
|
57
|
-
start = toIsoDate(addDays(anchor, -179));
|
|
58
|
-
break;
|
|
59
|
-
case "last-365d":
|
|
60
|
-
end = anchorIso;
|
|
61
|
-
start = toIsoDate(addDays(anchor, -364));
|
|
62
|
-
break;
|
|
63
|
-
case "mtd":
|
|
64
|
-
end = anchorIso;
|
|
65
|
-
start = toIsoDate(new Date(Date.UTC(anchor.getUTCFullYear(), anchor.getUTCMonth(), 1)));
|
|
66
|
-
break;
|
|
67
|
-
case "ytd":
|
|
68
|
-
end = anchorIso;
|
|
69
|
-
start = toIsoDate(new Date(Date.UTC(anchor.getUTCFullYear(), 0, 1)));
|
|
70
|
-
break;
|
|
71
|
-
case "custom":
|
|
72
|
-
if (!opts.start || !opts.end) throw new Error("resolveWindow: preset=custom requires start and end");
|
|
73
|
-
start = opts.start;
|
|
74
|
-
end = opts.end;
|
|
75
|
-
}
|
|
76
|
-
const days = daysBetween(start, end);
|
|
77
|
-
const result = {
|
|
78
|
-
start,
|
|
79
|
-
end,
|
|
80
|
-
days
|
|
81
|
-
};
|
|
82
|
-
const mode = opts.comparison ?? "none";
|
|
83
|
-
if (mode === "prev-period") {
|
|
84
|
-
const prevEnd = toIsoDate(addDays(parseIso(start), -1));
|
|
85
|
-
result.comparison = {
|
|
86
|
-
start: toIsoDate(addDays(parseIso(prevEnd), -(days - 1))),
|
|
87
|
-
end: prevEnd
|
|
88
|
-
};
|
|
89
|
-
} else if (mode === "yoy") {
|
|
90
|
-
const prevEnd = toIsoDate(addDays(parseIso(end), -365));
|
|
91
|
-
result.comparison = {
|
|
92
|
-
start: toIsoDate(addDays(parseIso(start), -365)),
|
|
93
|
-
end: prevEnd
|
|
94
|
-
};
|
|
95
|
-
}
|
|
96
|
-
return result;
|
|
97
|
-
}
|
|
98
24
|
const DEFAULT_FILL = {
|
|
99
25
|
clicks: 0,
|
|
100
26
|
impressions: 0,
|
package/dist/profile.d.mts
CHANGED
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gscdump/engine",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "3.
|
|
4
|
+
"version": "3.2.0",
|
|
5
5
|
"description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -41,6 +41,11 @@
|
|
|
41
41
|
"import": "./dist/planner.mjs",
|
|
42
42
|
"default": "./dist/planner.mjs"
|
|
43
43
|
},
|
|
44
|
+
"./profile": {
|
|
45
|
+
"types": "./dist/profile.d.mts",
|
|
46
|
+
"import": "./dist/profile.mjs",
|
|
47
|
+
"default": "./dist/profile.mjs"
|
|
48
|
+
},
|
|
44
49
|
"./schema": {
|
|
45
50
|
"types": "./dist/schema.d.mts",
|
|
46
51
|
"import": "./dist/schema.mjs",
|
|
@@ -186,9 +191,9 @@
|
|
|
186
191
|
"dependencies": {
|
|
187
192
|
"drizzle-orm": "1.0.0-rc.4",
|
|
188
193
|
"proper-lockfile": "^4.1.2",
|
|
189
|
-
"@gscdump/
|
|
190
|
-
"
|
|
191
|
-
"gscdump": "^3.
|
|
194
|
+
"@gscdump/lakehouse": "^3.2.0",
|
|
195
|
+
"gscdump": "^3.2.0",
|
|
196
|
+
"@gscdump/contracts": "^3.2.0"
|
|
192
197
|
},
|
|
193
198
|
"devDependencies": {
|
|
194
199
|
"@duckdb/duckdb-wasm": "1.33.1-dev57.0",
|