@gscdump/engine-sqlite 0.6.3 → 0.7.5

package/dist/index.mjs

@@ -1,9 +1,10 @@
 import { assertSchemaInSync, compileSqlite, compileSqlite as compileSqlite$1, createResolverAdapter, createSqlQuerySource } from "@gscdump/engine/resolver";
-import { and, eq, gte, lte, sql, sql as sql$1 } from "drizzle-orm";
-import { integer, real, sqliteTable, text } from "drizzle-orm/sqlite-core";
+import { and, and as and$1, eq, eq as eq$1, gte, inArray, isNotNull, isNull, lt, lte, lte as lte$1, or, sql, sql as sql$1 } from "drizzle-orm";
+import { index, integer, primaryKey, real, sqliteTable, text, unique } from "drizzle-orm/sqlite-core";
+import { inferSearchType } from "@gscdump/engine";
 import { createScopedHelpers } from "@gscdump/engine/scope";
 import { drizzle } from "drizzle-orm/sqlite-proxy";
-import { resolveWindow } from "@gscdump/analysis/period";
+import { resolveWindow } from "@gscdump/engine/period";
 function metricCols() {
 	return {
 		clicks: integer("clicks").notNull().default(0),
@@ -121,6 +122,346 @@ function aggCtr(t) {
 function aggPosition(t) {
 	return sql$1`SUM(${t.sum_position}) / NULLIF(SUM(${t.impressions}), 0) + 1`;
 }
+const r2Manifest = sqliteTable("r2_manifest", {
+	id: text("id").primaryKey(),
+	userId: integer("user_id").notNull(),
+	siteId: text("site_id"),
+	table: text("table").notNull(),
+	partition: text("partition").notNull(),
+	objectKey: text("object_key").notNull(),
+	rowCount: integer("row_count").notNull().default(0),
+	bytes: integer("bytes").notNull().default(0),
+	createdAt: integer("created_at").notNull(),
+	retiredAt: integer("retired_at"),
+	tier: text("tier"),
+	searchType: text("search_type"),
+	schemaVersion: integer("schema_version")
+}, (t) => [
+	index("idx_r2_manifest_live").on(t.userId, t.siteId, t.table, t.partition, t.retiredAt),
+	index("idx_r2_manifest_retired").on(t.retiredAt),
+	index("idx_r2_manifest_tier").on(t.userId, t.siteId, t.table, t.tier, t.retiredAt),
+	unique("r2_manifest_object_key_unique").on(t.objectKey)
+]);
+const r2WriteErrors = sqliteTable("r2_write_errors", {
+	id: text("id").primaryKey(),
+	userId: integer("user_id").notNull(),
+	siteId: text("site_id"),
+	table: text("table"),
+	date: text("date"),
+	error: text("error").notNull(),
+	createdAt: integer("created_at").notNull().default(sql$1`(unixepoch())`)
+}, (t) => [index("idx_r2_write_errors_user").on(t.userId, t.createdAt), index("idx_r2_write_errors_created").on(t.createdAt)]);
+const r2ShadowDiffs = sqliteTable("r2_shadow_diffs", {
+	id: text("id").primaryKey(),
+	userId: integer("user_id").notNull(),
+	siteId: text("site_id"),
+	endpoint: text("endpoint").notNull(),
+	diff: text("diff").notNull(),
+	createdAt: integer("created_at").notNull().default(sql$1`(unixepoch())`)
+}, (t) => [index("idx_r2_shadow_diffs_user").on(t.userId, t.createdAt), index("idx_r2_shadow_diffs_endpoint").on(t.endpoint, t.createdAt)]);
+const r2Locks = sqliteTable("r2_locks", {
+	scope: text("scope").primaryKey(),
+	holderId: text("holder_id").notNull(),
+	acquiredAt: integer("acquired_at").notNull(),
+	expiresAt: integer("expires_at").notNull()
+}, (t) => [index("idx_r2_locks_expires").on(t.expiresAt)]);
+const r2Watermarks = sqliteTable("r2_watermarks", {
+	userId: integer("user_id").notNull(),
+	siteId: text("site_id").notNull().default(""),
+	table: text("table").notNull(),
+	newestDateSynced: text("newest_date_synced").notNull(),
+	oldestDateSynced: text("oldest_date_synced").notNull(),
+	lastSyncAt: integer("last_sync_at").notNull()
+}, (t) => [primaryKey({ columns: [
+	t.userId,
+	t.siteId,
+	t.table
+] })]);
+const r2SyncStates = sqliteTable("r2_sync_states", {
+	userId: integer("user_id").notNull(),
+	siteId: text("site_id").notNull().default(""),
+	table: text("table").notNull(),
+	date: text("date").notNull(),
+	searchType: text("search_type").notNull().default(""),
+	state: text("state", { enum: [
+		"pending",
+		"inflight",
+		"done",
+		"failed"
+	] }).notNull(),
+	updatedAt: integer("updated_at").notNull(),
+	attempts: integer("attempts").notNull().default(0),
+	error: text("error")
+}, (t) => [primaryKey({ columns: [
+	t.userId,
+	t.siteId,
+	t.table,
+	t.date,
+	t.searchType
+] }), index("idx_r2_sync_states_state").on(t.state)]);
+function toRow(e) {
+	return {
+		id: e.objectKey,
+		userId: Number(e.userId),
+		siteId: e.siteId ?? null,
+		table: e.table,
+		partition: e.partition,
+		objectKey: e.objectKey,
+		rowCount: e.rowCount,
+		bytes: e.bytes,
+		createdAt: e.createdAt,
+		retiredAt: e.retiredAt ?? null,
+		tier: e.tier ?? null,
+		searchType: e.searchType ?? null,
+		schemaVersion: e.schemaVersion ?? null
+	};
+}
+function fromRow(r) {
+	return {
+		userId: String(r.userId),
+		siteId: r.siteId ?? void 0,
+		table: r.table,
+		partition: r.partition,
+		objectKey: r.objectKey,
+		rowCount: r.rowCount,
+		bytes: r.bytes,
+		createdAt: r.createdAt,
+		retiredAt: r.retiredAt ?? void 0,
+		...r.tier !== null ? { tier: r.tier } : {},
+		...r.searchType !== null ? { searchType: r.searchType } : {},
+		...r.schemaVersion !== null ? { schemaVersion: r.schemaVersion } : {}
+	};
+}
+const siteIdOf = (s) => s ?? "";
+const searchTypeOf = (s) => s === void 0 || s === "web" ? "" : s;
+const LOCK_TTL_MS = 3e4;
+const LOCK_ACQUIRE_TIMEOUT_MS = 5e3;
+const LOCK_RETRY_MIN_MS = 25;
+const LOCK_RETRY_MAX_MS = 150;
+function lockScopeKey(scope) {
+	return `${scope.userId}|${siteIdOf(scope.siteId)}|${scope.table}|${scope.partition}`;
+}
+function jitterDelay() {
+	return LOCK_RETRY_MIN_MS + Math.floor(Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS));
+}
+function tierMatchCond(target) {
+	const explicit = eq$1(r2Manifest.tier, target);
+	if (target === "raw") return or(explicit, and$1(isNull(r2Manifest.tier), sql$1`${r2Manifest.partition} LIKE 'daily/%'`));
+	if (target === "d30") return or(explicit, and$1(isNull(r2Manifest.tier), sql$1`${r2Manifest.partition} LIKE 'monthly/%'`));
+	return explicit;
+}
+function createD1ManifestStore(db) {
+	async function listByFilter(filter, liveOnly) {
+		const conds = [eq$1(r2Manifest.userId, Number(filter.userId))];
+		if (liveOnly) conds.push(isNull(r2Manifest.retiredAt));
+		if (filter.siteId !== void 0) conds.push(eq$1(r2Manifest.siteId, filter.siteId));
+		if (filter.table !== void 0) conds.push(eq$1(r2Manifest.table, filter.table));
+		if (filter.partitions && filter.partitions.length > 0) conds.push(inArray(r2Manifest.partition, filter.partitions));
+		if (filter.tier !== void 0) {
+			const cond = tierMatchCond(filter.tier);
+			if (cond) conds.push(cond);
+		}
+		return (await db.select().from(r2Manifest).where(and$1(...conds))).map(fromRow);
+	}
+	const listLive = (filter) => listByFilter(filter, true);
+	const listAll = (filter) => listByFilter(filter, false);
+	async function registerVersions(newEntries, superseding) {
+		const supersededAt = newEntries[0]?.createdAt ?? Date.now();
+		const statements = [];
+		if (superseding && superseding.length > 0) {
+			const keys = superseding.map((s) => s.objectKey);
+			const CHUNK = 90;
+			for (let i = 0; i < keys.length; i += CHUNK) {
+				const slice = keys.slice(i, i + CHUNK);
+				statements.push(db.update(r2Manifest).set({ retiredAt: supersededAt }).where(and$1(inArray(r2Manifest.objectKey, slice), isNull(r2Manifest.retiredAt))));
+			}
+		}
+		for (const e of newEntries) statements.push(db.insert(r2Manifest).values(toRow(e)).onConflictDoUpdate({
+			target: r2Manifest.objectKey,
+			set: {
+				userId: sql$1`excluded.user_id`,
+				siteId: sql$1`excluded.site_id`,
+				table: sql$1`excluded."table"`,
+				partition: sql$1`excluded.partition`,
+				rowCount: sql$1`excluded.row_count`,
+				bytes: sql$1`excluded.bytes`,
+				createdAt: sql$1`excluded.created_at`,
+				retiredAt: sql$1`excluded.retired_at`,
+				tier: sql$1`excluded.tier`,
+				searchType: sql$1`excluded.search_type`,
+				schemaVersion: sql$1`excluded.schema_version`
+			}
+		}));
+		if (statements.length === 0) return;
+		const BATCH_LIMIT = 95;
+		for (let i = 0; i < statements.length; i += BATCH_LIMIT) {
+			const chunk = statements.slice(i, i + BATCH_LIMIT);
+			await db.batch(chunk);
+		}
+	}
+	async function listRetired(olderThan) {
+		return (await db.select().from(r2Manifest).where(and$1(isNotNull(r2Manifest.retiredAt), lte$1(r2Manifest.retiredAt, olderThan)))).map(fromRow);
+	}
+	async function deleteEntries(entries) {
+		if (entries.length === 0) return;
+		const keys = entries.map((e) => e.objectKey);
+		const CHUNK = 90;
+		for (let i = 0; i < keys.length; i += CHUNK) await db.delete(r2Manifest).where(inArray(r2Manifest.objectKey, keys.slice(i, i + CHUNK)));
+	}
+	async function getWatermarks(filter) {
+		const conds = [eq$1(r2Watermarks.userId, Number(filter.userId))];
+		if (filter.siteId !== void 0) conds.push(eq$1(r2Watermarks.siteId, filter.siteId));
+		if (filter.table !== void 0) conds.push(eq$1(r2Watermarks.table, filter.table));
+		return (await db.select().from(r2Watermarks).where(and$1(...conds))).map((r) => ({
+			userId: String(r.userId),
+			siteId: r.siteId === "" ? void 0 : r.siteId,
+			table: r.table,
+			newestDateSynced: r.newestDateSynced,
+			oldestDateSynced: r.oldestDateSynced,
+			lastSyncAt: r.lastSyncAt
+		}));
+	}
+	async function bumpWatermark(scope, date, at) {
+		const lastSyncAt = at ?? Date.now();
+		await db.insert(r2Watermarks).values({
+			userId: Number(scope.userId),
+			siteId: siteIdOf(scope.siteId),
+			table: scope.table,
+			newestDateSynced: date,
+			oldestDateSynced: date,
+			lastSyncAt
+		}).onConflictDoUpdate({
+			target: [
+				r2Watermarks.userId,
+				r2Watermarks.siteId,
+				r2Watermarks.table
+			],
+			set: {
+				newestDateSynced: sql$1`CASE WHEN excluded.newest_date_synced > newest_date_synced THEN excluded.newest_date_synced ELSE newest_date_synced END`,
+				oldestDateSynced: sql$1`CASE WHEN excluded.oldest_date_synced < oldest_date_synced THEN excluded.oldest_date_synced ELSE oldest_date_synced END`,
+				lastSyncAt: sql$1`excluded.last_sync_at`
+			}
+		}).run();
+	}
+	async function getSyncStates(filter) {
+		const conds = [eq$1(r2SyncStates.userId, Number(filter.userId))];
+		if (filter.siteId !== void 0) conds.push(eq$1(r2SyncStates.siteId, filter.siteId));
+		if (filter.table !== void 0) conds.push(eq$1(r2SyncStates.table, filter.table));
+		if (filter.state !== void 0) conds.push(eq$1(r2SyncStates.state, filter.state));
+		if (filter.searchType !== void 0) conds.push(eq$1(r2SyncStates.searchType, searchTypeOf(filter.searchType)));
+		return (await db.select().from(r2SyncStates).where(and$1(...conds))).map((r) => ({
+			userId: String(r.userId),
+			siteId: r.siteId === "" ? void 0 : r.siteId,
+			table: r.table,
+			date: r.date,
+			searchType: inferSearchType({ searchType: r.searchType === "" ? void 0 : r.searchType }),
+			state: r.state,
+			updatedAt: r.updatedAt,
+			attempts: r.attempts,
+			error: r.error ?? void 0
+		}));
+	}
+	async function setSyncState(scope, state, detail) {
+		const updatedAt = detail?.at ?? Date.now();
+		const errorText = detail?.error ?? null;
+		await db.insert(r2SyncStates).values({
+			userId: Number(scope.userId),
+			siteId: siteIdOf(scope.siteId),
+			table: scope.table,
+			date: scope.date,
+			searchType: searchTypeOf(scope.searchType),
+			state,
+			updatedAt,
+			attempts: state === "inflight" ? 1 : 0,
+			error: errorText
+		}).onConflictDoUpdate({
+			target: [
+				r2SyncStates.userId,
+				r2SyncStates.siteId,
+				r2SyncStates.table,
+				r2SyncStates.date,
+				r2SyncStates.searchType
+			],
+			set: {
+				state: sql$1`excluded.state`,
+				updatedAt: sql$1`excluded.updated_at`,
+				attempts: sql$1`CASE WHEN excluded.state = 'inflight' THEN attempts + 1 ELSE attempts END`,
+				error: sql$1`CASE
+          WHEN excluded.state = 'done' THEN NULL
+          WHEN excluded.state = 'inflight' THEN error
+          ELSE excluded.error
+        END`
+			}
+		}).run();
+	}
+	async function withLock(scope, fn) {
+		const key = lockScopeKey(scope);
+		const holderId = crypto.randomUUID();
+		const deadline = Date.now() + LOCK_ACQUIRE_TIMEOUT_MS;
+		while (true) {
+			const now = Date.now();
+			const expiresAt = now + LOCK_TTL_MS;
+			await db.insert(r2Locks).values({
+				scope: key,
+				holderId,
+				acquiredAt: now,
+				expiresAt
+			}).onConflictDoUpdate({
+				target: r2Locks.scope,
+				set: {
+					holderId: sql$1`excluded.holder_id`,
+					acquiredAt: sql$1`excluded.acquired_at`,
+					expiresAt: sql$1`excluded.expires_at`
+				},
+				setWhere: lt(r2Locks.expiresAt, now)
+			}).run();
+			if ((await db.select({ holderId: r2Locks.holderId }).from(r2Locks).where(eq$1(r2Locks.scope, key)).get())?.holderId === holderId) break;
+			if (Date.now() >= deadline) throw new Error(`withLock: timed out acquiring ${key} after ${LOCK_ACQUIRE_TIMEOUT_MS}ms`);
+			await new Promise((resolve) => setTimeout(resolve, jitterDelay()));
+		}
+		return await fn().finally(async () => {
+			await Promise.resolve(db.delete(r2Locks).where(and$1(eq$1(r2Locks.scope, key), eq$1(r2Locks.holderId, holderId))).run()).catch(() => {});
+		});
+	}
+	async function purgeTenant(filter) {
+		const userIdNum = Number(filter.userId);
+		const entriesCond = filter.siteId !== void 0 ? and$1(eq$1(r2Manifest.userId, userIdNum), eq$1(r2Manifest.siteId, filter.siteId)) : eq$1(r2Manifest.userId, userIdNum);
+		const watermarksCond = filter.siteId !== void 0 ? and$1(eq$1(r2Watermarks.userId, userIdNum), eq$1(r2Watermarks.siteId, filter.siteId)) : eq$1(r2Watermarks.userId, userIdNum);
+		const syncStatesCond = filter.siteId !== void 0 ? and$1(eq$1(r2SyncStates.userId, userIdNum), eq$1(r2SyncStates.siteId, filter.siteId)) : eq$1(r2SyncStates.userId, userIdNum);
+		const [entriesRows, watermarkRows, syncStateRows] = await Promise.all([
+			db.select({ n: sql$1`count(*)` }).from(r2Manifest).where(entriesCond).all(),
+			db.select({ n: sql$1`count(*)` }).from(r2Watermarks).where(watermarksCond).all(),
+			db.select({ n: sql$1`count(*)` }).from(r2SyncStates).where(syncStatesCond).all()
+		]);
+		const entriesRemoved = Number(entriesRows[0]?.n ?? 0);
+		const watermarksRemoved = Number(watermarkRows[0]?.n ?? 0);
+		const syncStatesRemoved = Number(syncStateRows[0]?.n ?? 0);
+		await db.batch([
+			db.delete(r2Manifest).where(entriesCond),
+			db.delete(r2Watermarks).where(watermarksCond),
+			db.delete(r2SyncStates).where(syncStatesCond)
+		]);
+		return {
+			entriesRemoved,
+			watermarksRemoved,
+			syncStatesRemoved
+		};
+	}
+	return {
+		listLive,
+		listAll,
+		registerVersion: (entry, superseding) => registerVersions([entry], superseding),
+		registerVersions,
+		listRetired,
+		delete: deleteEntries,
+		getWatermarks,
+		bumpWatermark,
+		getSyncStates,
+		setSyncState,
+		withLock,
+		purgeTenant
+	};
+}
 function createSqliteInsightRunner(opts) {
 	const { executor, logger, rowsAsArrays, schema: schemaOverride } = opts;
 	const callback = async (sql, params, method) => {
@@ -138,4 +479,4 @@ function createSqliteInsightRunner(opts) {
 	}) };
 }
 const { scopeFor, mergeScope } = createScopedHelpers(schema);
-export { aggClicks, aggCtr, aggImpressions, aggPosition, and, compileSqlite, createEngine, createSqliteInsightRunner, createSqliteResolverAdapter, createSqliteResolverAdapterFromExecutor, eq, gsc_countries, gsc_devices, gsc_keywords, gsc_page_keywords, gsc_pages, gte, lte, mergeScope, probeSqliteRegex, resolveWindow, schema, scopeFor, sql, sqliteResolverAdapter };
+export { aggClicks, aggCtr, aggImpressions, aggPosition, and, compileSqlite, createD1ManifestStore, createEngine, createSqliteInsightRunner, createSqliteResolverAdapter, createSqliteResolverAdapterFromExecutor, eq, gsc_countries, gsc_devices, gsc_keywords, gsc_page_keywords, gsc_pages, gte, lte, mergeScope, probeSqliteRegex, r2Locks, r2Manifest, r2ShadowDiffs, r2SyncStates, r2Watermarks, r2WriteErrors, resolveWindow, schema, scopeFor, sql, sqliteResolverAdapter };