@secondlayer/sdk 6.47.0 → 6.48.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1286 -1211
- package/dist/index.js +21 -5
- package/dist/index.js.map +5 -5
- package/dist/sinks/bun-sqlite.d.ts +158 -0
- package/dist/sinks/bun-sqlite.js +255 -0
- package/dist/sinks/bun-sqlite.js.map +12 -0
- package/dist/sinks/core.d.ts +195 -0
- package/dist/sinks/core.js +215 -0
- package/dist/sinks/core.js.map +11 -0
- package/dist/sinks/drizzle.d.ts +184 -0
- package/dist/sinks/drizzle.js +309 -0
- package/dist/sinks/drizzle.js.map +12 -0
- package/dist/sinks/kysely.d.ts +87 -2
- package/dist/sinks/kysely.js +91 -54
- package/dist/sinks/kysely.js.map +6 -5
- package/dist/sinks/testing.d.ts +183 -0
- package/dist/sinks/testing.js +194 -0
- package/dist/sinks/testing.js.map +10 -0
- package/dist/streams/index.d.ts +558 -482
- package/dist/streams/index.js +18 -3
- package/dist/streams/index.js.map +4 -4
- package/dist/streams/rows.d.ts +208 -208
- package/dist/streams/rows.js +1 -1
- package/dist/streams/rows.js.map +2 -2
- package/dist/subgraphs/index.d.ts +1044 -968
- package/dist/subgraphs/index.js +21 -5
- package/dist/subgraphs/index.js.map +5 -5
- package/dist/x402.d.ts +49 -49
- package/dist/x402.js +1 -1
- package/dist/x402.js.map +2 -2
- package/package.json +25 -3
package/dist/sinks/kysely.js
CHANGED
|
@@ -1,6 +1,3 @@
|
|
|
1
|
-
// src/sinks/kysely.ts
|
|
2
|
-
import { sql } from "kysely";
|
|
3
|
-
|
|
4
1
|
// src/errors.ts
|
|
5
2
|
var DOCS_BASE = "https://www.secondlayer.tools";
|
|
6
3
|
|
|
@@ -158,79 +155,119 @@ function isByoBreakingDetails(x) {
|
|
|
158
155
|
return Array.isArray(d.reasons) && !!plan && typeof plan === "object" && typeof plan.dropStatement === "string";
|
|
159
156
|
}
|
|
160
157
|
|
|
161
|
-
// src/sinks/
|
|
162
|
-
|
|
163
|
-
|
|
158
|
+
// src/sinks/core.ts
|
|
159
|
+
var DEFAULT_CHECKPOINT_TABLE = "sl_consumer_checkpoints";
|
|
160
|
+
function assertSqlIdentifier(name, label) {
|
|
161
|
+
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) {
|
|
162
|
+
throw new ValidationError(`${label}: invalid identifier "${name}" (letters, digits, underscores only).`, 400);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
function quoteIdent(name) {
|
|
166
|
+
return `"${name}"`;
|
|
167
|
+
}
|
|
168
|
+
function createSink(driver, options) {
|
|
169
|
+
const { label } = options;
|
|
170
|
+
const checkpointTable = options.checkpointTable ?? DEFAULT_CHECKPOINT_TABLE;
|
|
164
171
|
if (options.tables.length === 0) {
|
|
165
|
-
throw new ValidationError(
|
|
172
|
+
throw new ValidationError(`${label}: \`tables\` is empty — declare every table the handler writes, or reorg rollback is a no-op and orphaned rows persist forever.`, 400);
|
|
166
173
|
}
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
throw new ValidationError(`kyselySink: invalid identifier "${name}" (letters, digits, underscores only).`, 400);
|
|
170
|
-
}
|
|
171
|
-
return `"${name}"`;
|
|
172
|
-
};
|
|
173
|
-
const cp = ident(checkpointTable);
|
|
174
|
-
const heightCol = ident(options.height);
|
|
174
|
+
assertSqlIdentifier(checkpointTable, label);
|
|
175
|
+
assertSqlIdentifier(options.height, label);
|
|
175
176
|
for (const table of options.tables)
|
|
176
|
-
|
|
177
|
-
async function acquireLock(tx) {
|
|
178
|
-
const locked = await sql`
|
|
179
|
-
SELECT pg_try_advisory_xact_lock(hashtext(${options.id})) AS "ok"
|
|
180
|
-
`.execute(tx);
|
|
181
|
-
if (!locked.rows[0]?.ok) {
|
|
182
|
-
throw new ValidationError(`kyselySink: another consumer holds the "${options.id}" lock — two replicas of the same consumer would interleave commits. Stop the other instance or use a distinct id.`, 409);
|
|
183
|
-
}
|
|
184
|
-
}
|
|
185
|
-
async function upsertCursor(tx, cursor) {
|
|
186
|
-
await sql`
|
|
187
|
-
INSERT INTO ${sql.raw(cp)} (id, cursor) VALUES (${options.id}, ${cursor})
|
|
188
|
-
ON CONFLICT (id) DO UPDATE SET cursor = EXCLUDED.cursor
|
|
189
|
-
`.execute(tx);
|
|
190
|
-
}
|
|
177
|
+
assertSqlIdentifier(table, label);
|
|
191
178
|
return {
|
|
179
|
+
capabilities: options.capabilities,
|
|
192
180
|
async loadCursor() {
|
|
193
|
-
await
|
|
181
|
+
await driver.ensureCheckpointStore();
|
|
194
182
|
for (const table of options.tables) {
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
SELECT 1 FROM information_schema.columns
|
|
198
|
-
WHERE table_name = ${table} AND column_name = ${options.height}
|
|
199
|
-
) AS "ok"
|
|
200
|
-
`.execute(db);
|
|
201
|
-
if (!col.rows[0]?.ok) {
|
|
202
|
-
throw new ValidationError(`kyselySink: table "${table}" has no "${String(options.height)}" column — the height stamp is what makes reorg rollback possible. Add the column (or fix \`height\`).`, 400);
|
|
183
|
+
if (!await driver.hasColumn(table, options.height)) {
|
|
184
|
+
throw new ValidationError(`${label}: table "${table}" has no "${options.height}" column — the height stamp is what makes reorg rollback possible. Add the column (or fix \`height\`).`, 400);
|
|
203
185
|
}
|
|
204
186
|
}
|
|
205
|
-
|
|
206
|
-
SELECT cursor FROM ${sql.raw(cp)} WHERE id = ${options.id}
|
|
207
|
-
`.execute(db);
|
|
208
|
-
return row.rows[0]?.cursor ?? null;
|
|
187
|
+
return driver.readCursor();
|
|
209
188
|
},
|
|
210
189
|
async commitBatch(cursor, write) {
|
|
211
|
-
await
|
|
212
|
-
await acquireLock(tx);
|
|
190
|
+
await driver.transact(async (tx) => {
|
|
191
|
+
await driver.acquireLock?.(tx);
|
|
213
192
|
await write(tx);
|
|
214
|
-
await
|
|
193
|
+
await driver.writeCursor(tx, cursor);
|
|
215
194
|
});
|
|
216
195
|
},
|
|
217
196
|
async rollback(forkPointHeight, rewindCursor) {
|
|
218
|
-
await
|
|
219
|
-
await acquireLock(tx);
|
|
197
|
+
await driver.transact(async (tx) => {
|
|
198
|
+
await driver.acquireLock?.(tx);
|
|
220
199
|
for (const table of options.tables) {
|
|
221
|
-
await
|
|
222
|
-
DELETE FROM ${sql.raw(ident(table))}
|
|
223
|
-
WHERE ${sql.raw(heightCol)} >= ${forkPointHeight}
|
|
224
|
-
`.execute(tx);
|
|
200
|
+
await driver.deleteAtOrAbove(tx, table, forkPointHeight);
|
|
225
201
|
}
|
|
226
|
-
await
|
|
202
|
+
await driver.writeCursor(tx, rewindCursor);
|
|
227
203
|
});
|
|
228
204
|
}
|
|
229
205
|
};
|
|
230
206
|
}
|
|
207
|
+
|
|
208
|
+
// src/sinks/kysely.ts
|
|
209
|
+
import { sql } from "kysely";
|
|
210
|
+
function kyselySink(db, options) {
|
|
211
|
+
const checkpointTable = options.checkpointTable ?? DEFAULT_CHECKPOINT_TABLE;
|
|
212
|
+
const cp = quoteIdent(checkpointTable);
|
|
213
|
+
const driver = {
|
|
214
|
+
transact: (fn) => db.transaction().execute(fn),
|
|
215
|
+
async ensureCheckpointStore() {
|
|
216
|
+
try {
|
|
217
|
+
await sql.raw(`CREATE TABLE IF NOT EXISTS ${cp} (id text PRIMARY KEY, cursor text NOT NULL)`).execute(db);
|
|
218
|
+
} catch (err) {
|
|
219
|
+
const code = err?.code;
|
|
220
|
+
if (code !== "42P07" && code !== "23505")
|
|
221
|
+
throw err;
|
|
222
|
+
}
|
|
223
|
+
},
|
|
224
|
+
async readCursor() {
|
|
225
|
+
const row = await sql`
|
|
226
|
+
SELECT cursor FROM ${sql.raw(cp)} WHERE id = ${options.id}
|
|
227
|
+
`.execute(db);
|
|
228
|
+
return row.rows[0]?.cursor ?? null;
|
|
229
|
+
},
|
|
230
|
+
async writeCursor(tx, cursor) {
|
|
231
|
+
await sql`
|
|
232
|
+
INSERT INTO ${sql.raw(cp)} (id, cursor) VALUES (${options.id}, ${cursor})
|
|
233
|
+
ON CONFLICT (id) DO UPDATE SET cursor = EXCLUDED.cursor
|
|
234
|
+
`.execute(tx);
|
|
235
|
+
},
|
|
236
|
+
async deleteAtOrAbove(tx, table, height) {
|
|
237
|
+
await sql`
|
|
238
|
+
DELETE FROM ${sql.raw(quoteIdent(table))}
|
|
239
|
+
WHERE ${sql.raw(quoteIdent(options.height))} >= ${height}
|
|
240
|
+
`.execute(tx);
|
|
241
|
+
},
|
|
242
|
+
async hasColumn(table, column) {
|
|
243
|
+
const col = await sql`
|
|
244
|
+
SELECT EXISTS (
|
|
245
|
+
SELECT 1 FROM information_schema.columns
|
|
246
|
+
WHERE table_name = ${table} AND column_name = ${column}
|
|
247
|
+
) AS "ok"
|
|
248
|
+
`.execute(db);
|
|
249
|
+
return col.rows[0]?.ok ?? false;
|
|
250
|
+
},
|
|
251
|
+
async acquireLock(tx) {
|
|
252
|
+
const locked = await sql`
|
|
253
|
+
SELECT pg_try_advisory_xact_lock(hashtextextended(${options.id}, 0)) AS "ok"
|
|
254
|
+
`.execute(tx);
|
|
255
|
+
if (!locked.rows[0]?.ok) {
|
|
256
|
+
throw new ValidationError(`kyselySink: another consumer holds the "${options.id}" lock — two replicas of the same consumer would interleave commits. Stop the other instance or use a distinct id.`, 409);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
};
|
|
260
|
+
return createSink(driver, {
|
|
261
|
+
label: "kyselySink",
|
|
262
|
+
id: options.id,
|
|
263
|
+
tables: options.tables,
|
|
264
|
+
height: options.height,
|
|
265
|
+
checkpointTable
|
|
266
|
+
});
|
|
267
|
+
}
|
|
231
268
|
export {
|
|
232
269
|
kyselySink
|
|
233
270
|
};
|
|
234
271
|
|
|
235
|
-
//# debugId=
|
|
272
|
+
//# debugId=475014242F550C1264756E2164756E21
|
|
236
273
|
//# sourceMappingURL=kysely.js.map
|
package/dist/sinks/kysely.js.map
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
|
-
"sources": ["
|
|
3
|
+
"sources": ["src/errors.ts", "src/sinks/core.ts", "src/sinks/kysely.ts"],
|
|
4
4
|
"sourcesContent": [
|
|
5
|
-
"import
|
|
6
|
-
"import
|
|
5
|
+
"import type { ByoBreakingChangeDetails } from \"@secondlayer/shared/errors\";\n\nexport type { ByoBreakingChangeDetails };\n\nconst DOCS_BASE = \"https://www.secondlayer.tools\";\n\n/** Options accepted by every error in the family. */\nexport interface SecondLayerErrorOptions {\n\tcause?: Error;\n\tdetails?: string;\n\t/** Stable machine-readable code (e.g. `UPGRADE_REQUIRED`, `BYO_BREAKING_CHANGE`). */\n\tcode?: string;\n\t/** Docs page that explains this failure and its fix. */\n\tdocsUrl?: string;\n\t/** Extra context lines appended to the message (DDL to run, next steps). */\n\tmetaMessages?: string[];\n\t/** Whether retrying the SAME request can succeed (429/5xx/network). */\n\tretryable?: boolean;\n\t/** Parsed `Retry-After`, in seconds — set on rate limits that carry it. */\n\tretryAfterSeconds?: number;\n}\n\n/**\n * Root of the SDK error family. Implements the same protocol as `BaseError`\n * in `@secondlayer/stacks` (`shortMessage`, `cause`, `toJSON`) so a failed\n * chain read and a failed platform read serialize and introspect identically.\n * It intentionally does NOT extend that class: `BaseError.details` is a\n * `string`, while {@link ByoBreakingChangeError.details} is the documented\n * structured migration plan — and cross-package `instanceof` is unreliable\n * under per-package bundling anyway (match on `code`/`name` across packages).\n *\n * `retryable` is the signal the consume loops act on: 429/5xx/network are\n * retryable, 4xx and body-serialization failures are not.\n */\nexport class SecondLayerError extends Error {\n\toverride name = \"SecondLayerError\";\n\t/** The one-line failure, without the appended context blocks. */\n\tshortMessage: string;\n\tcode?: string;\n\tdocsUrl?: string;\n\tmetaMessages?: string[];\n\tretryable: boolean;\n\tretryAfterSeconds?: number;\n\n\tconstructor(shortMessage: string, options: SecondLayerErrorOptions = {}) {\n\t\tconst detailBlocks = [\n\t\t\t...(options.metaMessages ?? []),\n\t\t\t...(options.details ? [options.details] : []),\n\t\t\t...(options.docsUrl ? [`Docs: ${options.docsUrl}`] : []),\n\t\t];\n\t\tconst message = [\n\t\t\tshortMessage,\n\t\t\tdetailBlocks.length > 0 ? `\\n${detailBlocks.join(\"\\n\")}` : \"\",\n\t\t].join(\"\");\n\t\tsuper(message, options.cause ? { cause: options.cause } : undefined);\n\t\tthis.shortMessage = shortMessage;\n\t\tthis.code = options.code;\n\t\tthis.docsUrl = options.docsUrl;\n\t\tthis.metaMessages = options.metaMessages;\n\t\tthis.retryable = options.retryable ?? false;\n\t\tthis.retryAfterSeconds = options.retryAfterSeconds;\n\t}\n\n\t/**\n\t * Walk the cause chain. With a predicate, returns the first error matching\n\t * it (or `null`); without one, returns the deepest cause. Mirrors viem's\n\t * `error.walk()` so cross-library error handling reads the same.\n\t */\n\twalk(fn?: (err: unknown) => boolean): unknown {\n\t\t// biome-ignore lint/suspicious/noExplicitAny: cause chains are untyped by nature\n\t\tlet err: any = this;\n\t\tlet last: unknown = this;\n\t\twhile (err != null) {\n\t\t\tif (fn?.(err)) return err;\n\t\t\tlast = err;\n\t\t\terr = err.cause;\n\t\t}\n\t\treturn fn ? null : last;\n\t}\n\n\ttoJSON(): {\n\t\tname: string;\n\t\tmessage: string;\n\t\tshortMessage: string;\n\t\tcause: string | undefined;\n\t\tcode: string | undefined;\n\t\tdocsUrl: string | undefined;\n\t\tretryable: boolean;\n\t\tretryAfterSeconds: number | undefined;\n\t} {\n\t\treturn {\n\t\t\tname: this.name,\n\t\t\tmessage: this.message,\n\t\t\tshortMessage: this.shortMessage,\n\t\t\tcause: this.cause instanceof Error ? this.cause.message : undefined,\n\t\t\tcode: this.code,\n\t\t\tdocsUrl: this.docsUrl,\n\t\t\tretryable: this.retryable,\n\t\t\tretryAfterSeconds: this.retryAfterSeconds,\n\t\t};\n\t}\n}\n\n/** Retryability of an HTTP status: 429 and 5xx are transient. */\nfunction statusRetryable(status: number): boolean {\n\treturn status === 429 || status >= 500;\n}\n\n/** Docs page for a status/code pair, when one exists. */\nfunction docsFor(status: number, code?: string): string | undefined {\n\tif (code === \"UPGRADE_REQUIRED\" || code === \"PLAN_REQUIRED\" || status === 402)\n\t\treturn `${DOCS_BASE}/docs/authentication#pay-as-you-go-credits`;\n\tif (status === 429 || status === 401)\n\t\treturn `${DOCS_BASE}/docs/authentication`;\n\treturn undefined;\n}\n\n/**\n * Error thrown by {@link SecondLayer} when an API request fails.\n * Includes the HTTP status code for programmatic error handling.\n *\n * @example\n * ```ts\n * try {\n * await client.subgraphs.get(\"my-subgraph\");\n * } catch (err) {\n * if (err instanceof ApiError && err.status === 404) {\n * console.log(\"Subgraph not found\");\n * }\n * }\n * ```\n */\nexport class ApiError extends SecondLayerError {\n\tconstructor(\n\t\t/** HTTP status code (0 for network errors). */\n\t\tpublic status: number,\n\t\tmessage: string,\n\t\t/** Raw response body (parsed JSON if possible) — preserved for callers that need error details. */\n\t\tpublic body?: unknown,\n\t\t/** Stable machine-readable code from the API's `{error, code}` error envelope. */\n\t\tcode?: string,\n\t\toptions: SecondLayerErrorOptions = {},\n\t) {\n\t\tsuper(message, {\n\t\t\tcode,\n\t\t\tretryable: options.retryable ?? statusRetryable(status),\n\t\t\tdocsUrl: options.docsUrl ?? docsFor(status, code),\n\t\t\t...(options.cause ? { cause: options.cause } : {}),\n\t\t\t...(options.details ? { details: options.details } : {}),\n\t\t\t...(options.metaMessages ? { metaMessages: options.metaMessages } : {}),\n\t\t\t...(options.retryAfterSeconds !== undefined\n\t\t\t\t? { retryAfterSeconds: options.retryAfterSeconds }\n\t\t\t\t: {}),\n\t\t});\n\t\tthis.name = \"ApiError\";\n\t}\n}\n\n/** Thrown on a 401 — by both the platform clients and Streams. */\nexport class AuthError extends ApiError {\n\toverride readonly status = 401 as const;\n\n\tconstructor(message = \"API key invalid or expired.\") {\n\t\tsuper(401, message);\n\t\tthis.name = \"AuthError\";\n\t}\n}\n\n/** Thrown on a 429 — by both the platform clients and Streams. `retryable`,\n * with `retryAfterSeconds` parsed from the `Retry-After` header when sent. */\nexport class RateLimitError extends ApiError {\n\toverride readonly status = 429 as const;\n\n\tconstructor(\n\t\tmessage = \"Rate limited. Try again later.\",\n\t\t/** Raw `Retry-After` header value (seconds or HTTP-date). */\n\t\treadonly retryAfter?: string,\n\t) {\n\t\tsuper(429, message, undefined, undefined, {\n\t\t\tretryable: true,\n\t\t\tretryAfterSeconds: parseRetryAfter(retryAfter),\n\t\t});\n\t\tthis.name = \"RateLimitError\";\n\t}\n}\n\n/** Thrown on a 4xx the caller can fix (bad cursor, bad params). Never retried. */\nexport class ValidationError extends ApiError {\n\tconstructor(message: string, status: number, body?: unknown) {\n\t\tsuper(status, message, body, undefined, { retryable: false });\n\t\tthis.name = \"ValidationError\";\n\t}\n}\n\n/** Parse a `Retry-After` header: delta-seconds or an HTTP-date. */\nexport function parseRetryAfter(value?: string | null): number | undefined {\n\tif (!value) return undefined;\n\tconst seconds = Number(value);\n\tif (Number.isFinite(seconds) && seconds >= 0) return seconds;\n\tconst date = Date.parse(value);\n\tif (!Number.isNaN(date)) return Math.max(0, (date - Date.now()) / 1000);\n\treturn undefined;\n}\n\n/**\n * Thrown on optimistic-concurrency conflict when a deploy supplies an\n * `expectedVersion` that no longer matches the server's stored version.\n */\nexport class VersionConflictError extends ApiError {\n\tconstructor(\n\t\tpublic currentVersion: string,\n\t\tpublic expectedVersion: string,\n\t\tmessage = `Version conflict: expected ${expectedVersion}, current ${currentVersion}`,\n\t) {\n\t\tsuper(409, message, { currentVersion, expectedVersion });\n\t\tthis.name = \"VersionConflictError\";\n\t}\n}\n\n/**\n * Thrown when a BYO subgraph deploy is refused for a breaking schema change.\n * The deploy did NOT run — `details.plan` carries the DROP + rebuild DDL to run\n * manually on your own database, plus the breaking `reasons` and the `diff`.\n * The same DDL is surfaced on `metaMessages`, so a bare `console.error(err)`\n * already shows the operator what to run.\n *\n * @example\n * ```ts\n * try {\n * await client.subgraphs.deploy(bundle);\n * } catch (err) {\n * if (err instanceof ByoBreakingChangeError) {\n * console.log(err.details.plan.dropStatement);\n * console.log(err.details.plan.statements.join(\";\\n\"));\n * }\n * }\n * ```\n */\nexport class ByoBreakingChangeError extends ApiError {\n\treadonly details: ByoBreakingChangeDetails;\n\tconstructor(message: string, details: ByoBreakingChangeDetails) {\n\t\tsuper(422, message, details, \"BYO_BREAKING_CHANGE\", {\n\t\t\tmetaMessages: [\n\t\t\t\t\"Review and run manually on your database:\",\n\t\t\t\tdetails.plan.dropStatement,\n\t\t\t\t...details.plan.statements,\n\t\t\t],\n\t\t});\n\t\tthis.name = \"ByoBreakingChangeError\";\n\t\tthis.details = details;\n\t}\n}\n\n/** Narrow an unknown error body's `details` to {@link ByoBreakingChangeDetails}. */\nexport function isByoBreakingDetails(\n\tx: unknown,\n): x is ByoBreakingChangeDetails {\n\tif (!x || typeof x !== \"object\") return false;\n\tconst d = x as Record<string, unknown>;\n\tconst plan = d.plan as Record<string, unknown> | undefined;\n\treturn (\n\t\tArray.isArray(d.reasons) &&\n\t\t!!plan &&\n\t\ttypeof plan === \"object\" &&\n\t\ttypeof plan.dropStatement === \"string\"\n\t);\n}\n",
|
|
6
|
+
"import { ValidationError } from \"../errors.ts\";\nimport type { ConsumerSink } from \"./types.ts\";\n\n/** Default checkpoint table name, shared by every SQL-backed sink so two\n * sinks on the same database land in one table keyed by `id`. */\nexport const DEFAULT_CHECKPOINT_TABLE = \"sl_consumer_checkpoints\";\n\n/**\n * The driver surface a database adapter implements — the ~7 methods that\n * actually differ between SQL stores. Everything above them (the\n * commit/rollback call sequences, the guards, identifier validation, the\n * checkpoint schema) is policy owned by {@link createSink}, so a driver\n * cannot get the ordering wrong: it never sees a cursor write outside the\n * transaction that carries the rows, and never sees a delete without the\n * rewound cursor riding along.\n *\n * Identifier args (`table`, `column`) arrive pre-validated against\n * `[A-Za-z_][A-Za-z0-9_]*` — safe to interpolate after quoting (see\n * {@link quoteIdent}). Values (cursor, height, id) must still be bound as\n * parameters.\n */\nexport interface SinkDriver<Tx> {\n\t/** Run `fn` inside ONE transaction: begin, run, commit — a throw from\n\t * `fn` must roll the whole transaction back and re-throw. */\n\ttransact<T>(fn: (tx: Tx) => Promise<T>): Promise<T>;\n\t/** Create the checkpoint store if missing: `(id primary key, cursor)`.\n\t * Must be safe under concurrent first start (two consumers racing the\n\t * same CREATE). */\n\tensureCheckpointStore(): Promise<void>;\n\t/** The committed cursor for this sink's `id`, or `null` if none. */\n\treadCursor(): Promise<string | null>;\n\t/** Upsert this sink's cursor row inside `tx`. */\n\twriteCursor(tx: Tx, cursor: string): Promise<void>;\n\t/** Delete rows with `height column >= height` from `table`, inside `tx`. */\n\tdeleteAtOrAbove(tx: Tx, table: string, height: number): Promise<void>;\n\t/** Whether `table` exists AND carries `column` — the first-use check that\n\t * keeps reorg rollback from being a silent no-op. */\n\thasColumn(table: string, column: string): Promise<boolean>;\n\t/** Optional: assert this consumer is the only live writer for its `id`,\n\t * inside `tx`. Must FAIL LOUDLY (throw) when another writer holds it —\n\t * never block or interleave (contract invariant #13). */\n\tacquireLock?(tx: Tx): Promise<void>;\n}\n\n/** Options for {@link createSink} — the policy inputs, driver-agnostic.\n * Concrete sinks re-expose these with schema-typed `tables`/`height`. */\nexport interface CreateSinkOptions {\n\t/** Error-message prefix naming the concrete sink (e.g. `\"kyselySink\"`),\n\t * so a thrown guard points at the thing the user actually constructed. */\n\tlabel: string;\n\t/** Checkpoint identity AND concurrency key. */\n\tid: string;\n\t/** Rollback scope: on a reorg, rows at/above the fork point are deleted\n\t * from exactly these tables. */\n\ttables: readonly string[];\n\t/** The block-height stamp column, present on every declared table. */\n\theight: string;\n\t/** Checkpoint table name. Default {@link DEFAULT_CHECKPOINT_TABLE}. */\n\tcheckpointTable?: string;\n\t/** Forwarded to {@link ConsumerSink.capabilities}. */\n\tcapabilities?: ConsumerSink[\"capabilities\"];\n}\n\n/** Throw unless `name` is a bare SQL identifier (letters, digits,\n * underscores; no leading digit) — the guard that makes interpolating\n * table/column names into DDL and DELETE statements safe. */\nexport function assertSqlIdentifier(name: string, label: string): void {\n\tif (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) {\n\t\tthrow new ValidationError(\n\t\t\t`${label}: invalid identifier \"${name}\" (letters, digits, underscores only).`,\n\t\t\t400,\n\t\t);\n\t}\n}\n\n/** Double-quote a (pre-validated) identifier for interpolation. */\nexport function quoteIdent(name: string): string {\n\treturn `\"${name}\"`;\n}\n\n/**\n * Build a {@link ConsumerSink} from a {@link SinkDriver}: the portable ~90\n * lines every SQL sink otherwise re-implements — and re-risks. Owns the two\n * transaction sequences (begin → lock → write rows → write cursor;\n * begin → lock → delete `>=` fork → write rewound cursor), the empty-`tables`\n * guard, identifier validation, and the first-use height-column check.\n *\n * A driver implemented against this base cannot violate contract invariants\n * #4, #5, #8, #9, or #11 (see {@link ConsumerSink}) without breaking\n * `transact` itself — which is exactly what the conformance kit probes.\n */\nexport function createSink<Tx>(\n\tdriver: SinkDriver<Tx>,\n\toptions: CreateSinkOptions,\n): ConsumerSink<Tx> {\n\tconst { label } = options;\n\tconst checkpointTable = options.checkpointTable ?? DEFAULT_CHECKPOINT_TABLE;\n\tif (options.tables.length === 0) {\n\t\tthrow new ValidationError(\n\t\t\t`${label}: \\`tables\\` is empty — declare every table the handler writes, or reorg rollback is a no-op and orphaned rows persist forever.`,\n\t\t\t400,\n\t\t);\n\t}\n\tassertSqlIdentifier(checkpointTable, label);\n\tassertSqlIdentifier(options.height, label);\n\tfor (const table of options.tables) assertSqlIdentifier(table, label);\n\n\treturn {\n\t\tcapabilities: options.capabilities,\n\n\t\tasync loadCursor() {\n\t\t\tawait driver.ensureCheckpointStore();\n\t\t\t// First-use validation of the rollback precondition: every declared\n\t\t\t// table must exist and carry the height column. Failing here beats a\n\t\t\t// rollback that silently deletes nothing during a reorg.\n\t\t\tfor (const table of options.tables) {\n\t\t\t\tif (!(await driver.hasColumn(table, options.height))) {\n\t\t\t\t\tthrow new ValidationError(\n\t\t\t\t\t\t`${label}: table \"${table}\" has no \"${options.height}\" column — the height stamp is what makes reorg rollback possible. Add the column (or fix \\`height\\`).`,\n\t\t\t\t\t\t400,\n\t\t\t\t\t);\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn driver.readCursor();\n\t\t},\n\n\t\tasync commitBatch(cursor, write) {\n\t\t\tawait driver.transact(async (tx) => {\n\t\t\t\tawait driver.acquireLock?.(tx);\n\t\t\t\tawait write(tx);\n\t\t\t\tawait driver.writeCursor(tx, cursor);\n\t\t\t});\n\t\t},\n\n\t\tasync rollback(forkPointHeight, rewindCursor) {\n\t\t\tawait driver.transact(async (tx) => {\n\t\t\t\tawait driver.acquireLock?.(tx);\n\t\t\t\t// INCLUSIVE of the fork block: the new canonical chain re-supplies\n\t\t\t\t// it, and the consumer rewinds to re-read from `fork:0`.\n\t\t\t\tfor (const table of options.tables) {\n\t\t\t\t\tawait driver.deleteAtOrAbove(tx, table, forkPointHeight);\n\t\t\t\t}\n\t\t\t\tawait driver.writeCursor(tx, rewindCursor);\n\t\t\t});\n\t\t},\n\t};\n}\n",
|
|
7
|
+
"import { type Kysely, type Transaction, sql } from \"kysely\";\nimport { ValidationError } from \"../errors.ts\";\nimport {\n\tDEFAULT_CHECKPOINT_TABLE,\n\ttype SinkDriver,\n\tcreateSink,\n\tquoteIdent,\n} from \"./core.ts\";\nimport type { ConsumerSink } from \"./types.ts\";\n\n/**\n * Column present on EVERY declared table — `keyof` a union of row types is\n * the intersection of their keys, so a column missing from any one table is\n * a compile error at the `height:` option.\n */\ntype CommonColumn<DB, T extends keyof DB> = keyof DB[T] & string;\n\nexport interface KyselySinkOptions<DB, T extends keyof DB & string> {\n\t/** Checkpoint identity AND concurrency key: the cursor row's primary key,\n\t * and the input to `pg_try_advisory_xact_lock(hashtextextended(id, 0))`.\n\t * The lock is transaction-scoped: it stops two replicas from\n\t * INTERLEAVING inside a commit, but is released at each commit boundary\n\t * — it is NOT a fence, so two replicas alternating batches both pass.\n\t * Run exactly one consumer per id; the lock turns the racy overlap loud,\n\t * it does not make a second replica safe. */\n\tid: string;\n\t/** Rollback scope. On a reorg, rows AT OR ABOVE the fork point are deleted\n\t * from exactly these tables (inclusive `>=` — the new chain re-supplies\n\t * the fork block). A table you write but don't declare keeps orphaned\n\t * rows forever, so declare everything the handler touches. */\n\ttables: readonly T[];\n\t/** The block-height stamp column, present on every declared table (compile\n\t * and first-use checked). Height-stamp rollback is correct for\n\t * APPEND-ONLY projections (one row per event/call); aggregates that\n\t * mutate rows (balances, counters) can't be rolled back this way — use a\n\t * subgraph for those. */\n\theight: CommonColumn<DB, T>;\n\t/** Checkpoint table name. Default `sl_consumer_checkpoints` (created on\n\t * first use). */\n\tcheckpointTable?: string;\n}\n\n/**\n * A {@link ConsumerSink} over a Kysely Postgres database — keyed on the\n * QUERY BUILDER (not the driver) so the `DB` schema generic flows through:\n * `tables` is `(keyof DB)[]`, `height` must exist on every declared table,\n * and `ctx.tx` inside `onBatch` is a fully typed `Transaction<DB>`.\n *\n * What it owns (so your indexer doesn't):\n * - the checkpoint table + cursor load on start;\n * - rows AND cursor committed in ONE transaction per batch — a handler\n * throw aborts both, so a crashed batch is re-read on restart with no\n * gaps and no double-writes;\n * - reorg rollback: delete `>= fork_point_height` from every declared table\n * and commit the rewound cursor atomically (the crash-between-the-two-\n * writes gap is unrepresentable);\n * - a per-id advisory lock, so two replicas of the same consumer can't\n * interleave commits.\n *\n * Structurally it is a 7-method {@link SinkDriver} on the shared\n * `createSink` base (`@secondlayer/sdk/sinks/core`), which owns the\n * transaction sequences and guards — this file only knows how to speak\n * Postgres-flavored Kysely.\n *\n * ```ts\n * const sink = kyselySink(db, { id: \"sales\", tables: [\"sales\"], height: \"block_height\" });\n * await index.contractCalls.consume({\n * contractId: MARKETPLACE, functionName: \"purchase-asset\", fromHeight: 0,\n * sink,\n * onBatch: (calls, _env, ctx) =>\n * ctx.tx.insertInto(\"sales\").values(calls.map(toSale)).execute(),\n * });\n * ```\n */\nexport function kyselySink<DB, T extends keyof DB & string>(\n\tdb: Kysely<DB>,\n\toptions: KyselySinkOptions<DB, T>,\n): ConsumerSink<Transaction<DB>> {\n\tconst checkpointTable = options.checkpointTable ?? DEFAULT_CHECKPOINT_TABLE;\n\tconst cp = quoteIdent(checkpointTable);\n\n\tconst driver: SinkDriver<Transaction<DB>> = {\n\t\ttransact: (fn) => db.transaction().execute(fn),\n\n\t\tasync ensureCheckpointStore() {\n\t\t\ttry {\n\t\t\t\tawait sql\n\t\t\t\t\t.raw(\n\t\t\t\t\t\t`CREATE TABLE IF NOT EXISTS ${cp} (id text PRIMARY KEY, cursor text NOT NULL)`,\n\t\t\t\t\t)\n\t\t\t\t\t.execute(db);\n\t\t\t} catch (err) {\n\t\t\t\t// Two consumers racing the same first start: IF NOT EXISTS still\n\t\t\t\t// throws under concurrency (42P07 duplicate_table, or 23505 on the\n\t\t\t\t// pg_type catalog). Either way the table exists — the outcome we\n\t\t\t\t// were creating.\n\t\t\t\tconst code = (err as { code?: string } | null)?.code;\n\t\t\t\tif (code !== \"42P07\" && code !== \"23505\") throw err;\n\t\t\t}\n\t\t},\n\n\t\tasync readCursor() {\n\t\t\tconst row = await sql<{ cursor: string }>`\n\t\t\t\tSELECT cursor FROM ${sql.raw(cp)} WHERE id = ${options.id}\n\t\t\t`.execute(db);\n\t\t\treturn row.rows[0]?.cursor ?? null;\n\t\t},\n\n\t\tasync writeCursor(tx, cursor) {\n\t\t\t// Identifiers via validated sql.raw; VALUES via bound parameters.\n\t\t\tawait sql`\n\t\t\t\tINSERT INTO ${sql.raw(cp)} (id, cursor) VALUES (${options.id}, ${cursor})\n\t\t\t\tON CONFLICT (id) DO UPDATE SET cursor = EXCLUDED.cursor\n\t\t\t`.execute(tx);\n\t\t},\n\n\t\tasync deleteAtOrAbove(tx, table, height) {\n\t\t\t// Identifiers via validated sql.raw; the height bound as a parameter.\n\t\t\tawait sql`\n\t\t\t\tDELETE FROM ${sql.raw(quoteIdent(table))}\n\t\t\t\tWHERE ${sql.raw(quoteIdent(options.height as string))} >= ${height}\n\t\t\t`.execute(tx);\n\t\t},\n\n\t\tasync hasColumn(table, column) {\n\t\t\tconst col = await sql<{ ok: boolean }>`\n\t\t\t\tSELECT EXISTS (\n\t\t\t\t\tSELECT 1 FROM information_schema.columns\n\t\t\t\t\tWHERE table_name = ${table} AND column_name = ${column}\n\t\t\t\t) AS \"ok\"\n\t\t\t`.execute(db);\n\t\t\treturn col.rows[0]?.ok ?? false;\n\t\t},\n\n\t\tasync acquireLock(tx) {\n\t\t\t// hashtextextended (int8), not hashtext (int4): the 32-bit space\n\t\t\t// collides across distinct ids at real scale, and a collision reads\n\t\t\t// as a false \"another consumer\" 409 between unrelated sinks.\n\t\t\tconst locked = await sql<{ ok: boolean }>`\n\t\t\t\tSELECT pg_try_advisory_xact_lock(hashtextextended(${options.id}, 0)) AS \"ok\"\n\t\t\t`.execute(tx);\n\t\t\tif (!locked.rows[0]?.ok) {\n\t\t\t\tthrow new ValidationError(\n\t\t\t\t\t`kyselySink: another consumer holds the \"${options.id}\" lock — two replicas of the same consumer would interleave commits. Stop the other instance or use a distinct id.`,\n\t\t\t\t\t409,\n\t\t\t\t);\n\t\t\t}\n\t\t},\n\t};\n\n\treturn createSink(driver, {\n\t\tlabel: \"kyselySink\",\n\t\tid: options.id,\n\t\ttables: options.tables,\n\t\theight: options.height as string,\n\t\tcheckpointTable,\n\t});\n}\n"
|
|
7
8
|
],
|
|
8
|
-
"mappings": ";
|
|
9
|
-
"debugId": "
|
|
9
|
+
"mappings": ";AAIA,IAAM,YAAY;AAAA;AA8BX,MAAM,yBAAyB,MAAM;AAAA,EAClC,OAAO;AAAA,EAEhB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA,WAAW,CAAC,cAAsB,UAAmC,CAAC,GAAG;AAAA,IACxE,MAAM,eAAe;AAAA,MACpB,GAAI,QAAQ,gBAAgB,CAAC;AAAA,MAC7B,GAAI,QAAQ,UAAU,CAAC,QAAQ,OAAO,IAAI,CAAC;AAAA,MAC3C,GAAI,QAAQ,UAAU,CAAC,SAAS,QAAQ,SAAS,IAAI,CAAC;AAAA,IACvD;AAAA,IACA,MAAM,UAAU;AAAA,MACf;AAAA,MACA,aAAa,SAAS,IAAI;AAAA,EAAK,aAAa,KAAK;AAAA,CAAI,MAAM;AAAA,IAC5D,EAAE,KAAK,EAAE;AAAA,IACT,MAAM,SAAS,QAAQ,QAAQ,EAAE,OAAO,QAAQ,MAAM,IAAI,SAAS;AAAA,IACnE,KAAK,eAAe;AAAA,IACpB,KAAK,OAAO,QAAQ;AAAA,IACpB,KAAK,UAAU,QAAQ;AAAA,IACvB,KAAK,eAAe,QAAQ;AAAA,IAC5B,KAAK,YAAY,QAAQ,aAAa;AAAA,IACtC,KAAK,oBAAoB,QAAQ;AAAA;AAAA,EAQlC,IAAI,CAAC,IAAyC;AAAA,IAE7C,IAAI,MAAW;AAAA,IACf,IAAI,OAAgB;AAAA,IACpB,OAAO,OAAO,MAAM;AAAA,MACnB,IAAI,KAAK,GAAG;AAAA,QAAG,OAAO;AAAA,MACtB,OAAO;AAAA,MACP,MAAM,IAAI;AAAA,IACX;AAAA,IACA,OAAO,KAAK,OAAO;AAAA;AAAA,EAGpB,MAAM,GASJ;AAAA,IACD,OAAO;AAAA,MACN,MAAM,KAAK;AAAA,MACX,SAAS,KAAK;AAAA,MACd,cAAc,KAAK;AAAA,MACnB,OAAO,KAAK,iBAAiB,QAAQ,KAAK,MAAM,UAAU;AAAA,MAC1D,MAAM,KAAK;AAAA,MACX,SAAS,KAAK;AAAA,MACd,WAAW,KAAK;AAAA,MAChB,mBAAmB,KAAK;AAAA,IACzB;AAAA;AAEF;AAGA,SAAS,eAAe,CAAC,QAAyB;AAAA,EACjD,OAAO,WAAW,OAAO,UAAU;AAAA;AAIpC,SAAS,OAAO,CAAC,QAAgB,MAAmC;AAAA,EACnE,IAAI,SAAS,sBAAsB,SAAS,mBAAmB,WAAW;AAAA,IACzE,OAAO,GAAG;AAAA,EACX,IAAI,WAAW,OAAO,WAAW;AAAA,IAChC,OAAO,GAAG;AAAA,EACX;AAAA;AAAA;AAkBM,MAAM,iBAAiB,iBAAiB;AAAA,EAGtC;AAAA,EAGA;AAAA,EALR,WAAW,CAEH,QACP,SAEO,MAEP,MACA,UAAmC,CAAC,GACnC;AAAA,IACD,MAAM,SAAS;AAAA,MACd;AAAA,MACA,WAAW,QAAQ,aAAa,gBAAgB,MAAM;AAAA,MACtD,SAAS,QAAQ,WAAW,QAAQ,QAAQ,IAAI;AAAA,SAC5C,QAAQ,QAAQ,EAAE,OAAO,QAAQ,MAAM,IAAI,CAAC;AAAA,SAC5C,QAAQ,UAAU,EAAE,SAAS,QAAQ,QAAQ,IAAI,CAAC;AAAA,SAClD,QAAQ,eAAe,EAAE,cAAc,QAAQ,aAAa,IAAI,CAAC;AAAA,SACjE,QAAQ,sBAAsB,YAC/B,EAAE,mBAAmB,QAAQ,kBAAkB,IAC/C,CAAC;AAAA,IACL,CAAC;AAAA,IAlBM;AAAA,IAGA;AAAA,IAgBP,KAAK,OAAO;AAAA;AAEd;AAAA;AAGO,MAAM,kBAAkB,SAAS;AAAA,EACrB,SAAS;AAAA,EAE3B,WAAW,CAAC,UAAU,+BAA+B;AAAA,IACpD,MAAM,KAAK,OAAO;AAAA,IAClB,KAAK,OAAO;AAAA;AAEd;AAAA;AAIO,MAAM,uBAAuB,SAAS;AAAA,EAMlC;AAAA,EALQ,SAAS;AAAA,EAE3B,WAAW,CACV,UAAU,kCAED,YACR;AAAA,IACD,MAAM,KAAK,SAAS,WAAW,WAAW;AAAA,MACzC,WAAW;AAAA,MACX,mBAAmB,gBAAgB,UAAU;AAAA,IAC9C,CAAC;AAAA,IALQ;AAAA,IAMT,KAAK,OAAO;AAAA;AAEd;AAAA;AAGO,MAAM,wBAAwB,SAAS;AAAA,EAC7C,WAAW,CAAC,SAAiB,QAAgB,MAAgB;AAAA,IAC5D,MAAM,QAAQ,SAAS,MAAM,WAAW,EAAE,WAAW,MAAM,CAAC;AAAA,IAC5D,KAAK,OAAO;AAAA;AAEd;AAGO,SAAS,eAAe,CAAC,OAA2C;AAAA,EAC1E,IAAI,CAAC;AAAA,IAAO;AAAA,EACZ,MAAM,UAAU,OAAO,KAAK;AAAA,EAC5B,IAAI,OAAO,SAAS,OAAO,KAAK,WAAW;AAAA,IAAG,OAAO;AAAA,EACrD,MAAM,OAAO,KAAK,MAAM,KAAK;AAAA,EAC7B,IAAI,CAAC,OAAO,MAAM,IAAI;AAAA,IAAG,OAAO,KAAK,IAAI,IAAI,OAAO,KAAK,IAAI,KAAK,IAAI;AAAA,EACtE;AAAA;AAAA;AAOM,MAAM,6BAA6B,SAAS;AAAA,EAE1C;AAAA,EACA;AAAA,EAFR,WAAW,CACH,gBACA,iBACP,UAAU,8BAA8B,4BAA4B,kBACnE;AAAA,IACD,MAAM,KAAK,SAAS,EAAE,gBAAgB,gBAAgB,CAAC;AAAA,IAJhD;AAAA,IACA;AAAA,IAIP,KAAK,OAAO;AAAA;AAEd;AAAA;AAqBO,MAAM,+BAA+B,SAAS;AAAA,EAC3C;AAAA,EACT,WAAW,CAAC,SAAiB,SAAmC;AAAA,IAC/D,MAAM,KAAK,SAAS,SAAS,uBAAuB;AAAA,MACnD,cAAc;AAAA,QACb;AAAA,QACA,QAAQ,KAAK;AAAA,QACb,GAAG,QAAQ,KAAK;AAAA,MACjB;AAAA,IACD,CAAC;AAAA,IACD,KAAK,OAAO;AAAA,IACZ,KAAK,UAAU;AAAA;AAEjB;AAGO,SAAS,oBAAoB,CACnC,GACgC;AAAA,EAChC,IAAI,CAAC,KAAK,OAAO,MAAM;AAAA,IAAU,OAAO;AAAA,EACxC,MAAM,IAAI;AAAA,EACV,MAAM,OAAO,EAAE;AAAA,EACf,OACC,MAAM,QAAQ,EAAE,OAAO,KACvB,CAAC,CAAC,QACF,OAAO,SAAS,YAChB,OAAO,KAAK,kBAAkB;AAAA;;;ACnQzB,IAAM,2BAA2B;AA6DjC,SAAS,mBAAmB,CAAC,MAAc,OAAqB;AAAA,EACtE,IAAI,CAAC,2BAA2B,KAAK,IAAI,GAAG;AAAA,IAC3C,MAAM,IAAI,gBACT,GAAG,8BAA8B,8CACjC,GACD;AAAA,EACD;AAAA;AAIM,SAAS,UAAU,CAAC,MAAsB;AAAA,EAChD,OAAO,IAAI;AAAA;AAcL,SAAS,UAAc,CAC7B,QACA,SACmB;AAAA,EACnB,QAAQ,UAAU;AAAA,EAClB,MAAM,kBAAkB,QAAQ,mBAAmB;AAAA,EACnD,IAAI,QAAQ,OAAO,WAAW,GAAG;AAAA,IAChC,MAAM,IAAI,gBACT,GAAG,wIACH,GACD;AAAA,EACD;AAAA,EACA,oBAAoB,iBAAiB,KAAK;AAAA,EAC1C,oBAAoB,QAAQ,QAAQ,KAAK;AAAA,EACzC,WAAW,SAAS,QAAQ;AAAA,IAAQ,oBAAoB,OAAO,KAAK;AAAA,EAEpE,OAAO;AAAA,IACN,cAAc,QAAQ;AAAA,SAEhB,WAAU,GAAG;AAAA,MAClB,MAAM,OAAO,sBAAsB;AAAA,MAInC,WAAW,SAAS,QAAQ,QAAQ;AAAA,QACnC,IAAI,CAAE,MAAM,OAAO,UAAU,OAAO,QAAQ,MAAM,GAAI;AAAA,UACrD,MAAM,IAAI,gBACT,GAAG,iBAAiB,kBAAkB,QAAQ,gHAC9C,GACD;AAAA,QACD;AAAA,MACD;AAAA,MACA,OAAO,OAAO,WAAW;AAAA;AAAA,SAGpB,YAAW,CAAC,QAAQ,OAAO;AAAA,MAChC,MAAM,OAAO,SAAS,OAAO,OAAO;AAAA,QACnC,MAAM,OAAO,cAAc,EAAE;AAAA,QAC7B,MAAM,MAAM,EAAE;AAAA,QACd,MAAM,OAAO,YAAY,IAAI,MAAM;AAAA,OACnC;AAAA;AAAA,SAGI,SAAQ,CAAC,iBAAiB,cAAc;AAAA,MAC7C,MAAM,OAAO,SAAS,OAAO,OAAO;AAAA,QACnC,MAAM,OAAO,cAAc,EAAE;AAAA,QAG7B,WAAW,SAAS,QAAQ,QAAQ;AAAA,UACnC,MAAM,OAAO,gBAAgB,IAAI,OAAO,eAAe;AAAA,QACxD;AAAA,QACA,MAAM,OAAO,YAAY,IAAI,YAAY;AAAA,OACzC;AAAA;AAAA,EAEH;AAAA;;;ACjJD;AA0EO,SAAS,UAA2C,CAC1D,IACA,SACgC;AAAA,EAChC,MAAM,kBAAkB,QAAQ,mBAAmB;AAAA,EACnD,MAAM,KAAK,WAAW,eAAe;AAAA,EAErC,MAAM,SAAsC;AAAA,IAC3C,UAAU,CAAC,OAAO,GAAG,YAAY,EAAE,QAAQ,EAAE;AAAA,SAEvC,sBAAqB,GAAG;AAAA,MAC7B,IAAI;AAAA,QACH,MAAM,IACJ,IACA,8BAA8B,gDAC/B,EACC,QAAQ,EAAE;AAAA,QACX,OAAO,KAAK;AAAA,QAKb,MAAM,OAAQ,KAAkC;AAAA,QAChD,IAAI,SAAS,WAAW,SAAS;AAAA,UAAS,MAAM;AAAA;AAAA;AAAA,SAI5C,WAAU,GAAG;AAAA,MAClB,MAAM,MAAM,MAAM;AAAA,yBACI,IAAI,IAAI,EAAE,gBAAgB,QAAQ;AAAA,KACtD,QAAQ,EAAE;AAAA,MACZ,OAAO,IAAI,KAAK,IAAI,UAAU;AAAA;AAAA,SAGzB,YAAW,CAAC,IAAI,QAAQ;AAAA,MAE7B,MAAM;AAAA,kBACS,IAAI,IAAI,EAAE,0BAA0B,QAAQ,OAAO;AAAA;AAAA,KAEhE,QAAQ,EAAE;AAAA;AAAA,SAGP,gBAAe,CAAC,IAAI,OAAO,QAAQ;AAAA,MAExC,MAAM;AAAA,kBACS,IAAI,IAAI,WAAW,KAAK,CAAC;AAAA,YAC/B,IAAI,IAAI,WAAW,QAAQ,MAAgB,CAAC,QAAQ;AAAA,KAC3D,QAAQ,EAAE;AAAA;AAAA,SAGP,UAAS,CAAC,OAAO,QAAQ;AAAA,MAC9B,MAAM,MAAM,MAAM;AAAA;AAAA;AAAA,0BAGK,2BAA2B;AAAA;AAAA,KAEhD,QAAQ,EAAE;AAAA,MACZ,OAAO,IAAI,KAAK,IAAI,MAAM;AAAA;AAAA,SAGrB,YAAW,CAAC,IAAI;AAAA,MAIrB,MAAM,SAAS,MAAM;AAAA,wDACgC,QAAQ;AAAA,KAC3D,QAAQ,EAAE;AAAA,MACZ,IAAI,CAAC,OAAO,KAAK,IAAI,IAAI;AAAA,QACxB,MAAM,IAAI,gBACT,2CAA2C,QAAQ,wHACnD,GACD;AAAA,MACD;AAAA;AAAA,EAEF;AAAA,EAEA,OAAO,WAAW,QAAQ;AAAA,IACzB,OAAO;AAAA,IACP,IAAI,QAAQ;AAAA,IACZ,QAAQ,QAAQ;AAAA,IAChB,QAAQ,QAAQ;AAAA,IAChB;AAAA,EACD,CAAC;AAAA;",
|
|
10
|
+
"debugId": "475014242F550C1264756E2164756E21",
|
|
10
11
|
"names": []
|
|
11
12
|
}
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A consumer sink: the destination adapter a consume loop writes through.
|
|
3
|
+
*
|
|
4
|
+
* Without a sink, the user owns the three hard parts of a durable indexer —
|
|
5
|
+
* checkpoint persistence, rows+cursor atomicity, and reorg rollback — and
|
|
6
|
+
* the reasoning that makes them safe lives in doc comments. A sink owns all
|
|
7
|
+
* three: the loop loads its committed cursor, hands the handler a
|
|
8
|
+
* transaction, commits rows AND cursor atomically, and rolls back reorgs
|
|
9
|
+
* without any user code. Crucially, a sink also closes the silent-skip
|
|
10
|
+
* hazard where omitting `onReorg` meant reorgs were ignored forever: with a
|
|
11
|
+
* sink attached, rollback is unconditional.
|
|
12
|
+
*
|
|
13
|
+
* The interface is dependency-free on purpose: implementations with real
|
|
14
|
+
* database drivers live behind subpath exports (`@secondlayer/sdk/sinks/*`)
|
|
15
|
+
* so the root entry ships no DB dependency.
|
|
16
|
+
*
|
|
17
|
+
* ## The contract
|
|
18
|
+
*
|
|
19
|
+
* Binding on every implementation; the conformance kit
|
|
20
|
+
* (`@secondlayer/sdk/sinks/testing`) probes each one. Violations are SILENT
|
|
21
|
+
* in production — they surface as gaps or duplicates weeks later, never as
|
|
22
|
+
* errors at the violation site — which is why they are spelled out here and
|
|
23
|
+
* mechanically tested.
|
|
24
|
+
*
|
|
25
|
+
* What the loop guarantees the sink:
|
|
26
|
+
*
|
|
27
|
+
* 1. **Init-before-first-page.** `loadCursor` is called exactly once, before
|
|
28
|
+
* the first fetch — even when the caller passed an explicit `fromCursor`.
|
|
29
|
+
* It is the sink's init hook: create checkpoint storage, validate
|
|
30
|
+
* rollback preconditions.
|
|
31
|
+
* 2. **Deterministic replay.** Re-reading from the same cursor yields the
|
|
32
|
+
* same rows in the same order. `cursor` is therefore a valid idempotency
|
|
33
|
+
* key: an append-only sink can dedup on it with no contract change.
|
|
34
|
+
* 3. **At-least-once delivery.** A batch whose commit outcome was lost (crash
|
|
35
|
+
* after commit, before the loop observed it) is re-committed on restart
|
|
36
|
+
* with the SAME cursor and the same rows.
|
|
37
|
+
*
|
|
38
|
+
* What the sink must guarantee — `commitBatch`:
|
|
39
|
+
*
|
|
40
|
+
* 4. **Rows+cursor atomicity.** The handler's writes and `cursor` commit in
|
|
41
|
+
* ONE transaction. Committing them separately is the classic torn-batch
|
|
42
|
+
* bug: a crash between the two either re-delivers (duplicates) or skips
|
|
43
|
+
* (gap) depending on the order.
|
|
44
|
+
* 5. **Abort on throw.** A throw from `write` aborts the WHOLE transaction —
|
|
45
|
+
* neither rows nor cursor land — so a crashed batch is simply re-read.
|
|
46
|
+
* 6. **The lent transaction is the real one.** `write(tx)` receives the same
|
|
47
|
+
* live transaction the cursor commits in; every handler write must go
|
|
48
|
+
* through it. Writes outside `tx` escape atomicity AND rollback.
|
|
49
|
+
* 7. **Replay safety.** Committing the same cursor twice must not corrupt
|
|
50
|
+
* state (per #3 the rows are identical; upsert/dedup make it exact).
|
|
51
|
+
*
|
|
52
|
+
* What the sink must guarantee — `rollback`:
|
|
53
|
+
*
|
|
54
|
+
* 8. **Delete+rewind atomicity.** Undoing rows at/above the fork and
|
|
55
|
+
* committing `rewindCursor` happen in ONE transaction. A crash between
|
|
56
|
+
* them resumes above the fork and the deleted range is never re-read —
|
|
57
|
+
* the silent-gap bug.
|
|
58
|
+
* 9. **Inclusive fork point.** Undo AT OR ABOVE `forkPointHeight` (`>=`) —
|
|
59
|
+
* the new canonical chain re-supplies the fork block itself.
|
|
60
|
+
* 10. **Idempotent.** The loop may re-apply the same rollback after a crash
|
|
61
|
+
* (reorgs are deduped in memory only); a second application must be a
|
|
62
|
+
* harmless no-op.
|
|
63
|
+
* 11. **Scope by height, not cursor.** On a page reporting several forks the
|
|
64
|
+
* loop calls `rollback` once per fork: each call carries its own
|
|
65
|
+
* `forkPointHeight` but ALL carry the same `rewindCursor` (the lowest
|
|
66
|
+
* fork point). Derive the undo range from `forkPointHeight` only; a
|
|
67
|
+
* sink that derives it from `rewindCursor` under-deletes silently.
|
|
68
|
+
*
|
|
69
|
+
* Error semantics and concurrency:
|
|
70
|
+
*
|
|
71
|
+
* 12. **Throw, don't swallow; no internal retry.** The loop owns retry
|
|
72
|
+
* policy. A swallowed commit error advances the loop past unwritten
|
|
73
|
+
* data; an internal retry can double-apply around a partial failure.
|
|
74
|
+
* 13. **Single writer per checkpoint id.** The loop assumes one live
|
|
75
|
+
* consumer per checkpoint identity. A sink that can detect a second
|
|
76
|
+
* writer (e.g. a lock) must fail loudly — never block or interleave.
|
|
77
|
+
*/
|
|
78
|
+
interface ConsumerSink<Tx = unknown> {
|
|
79
|
+
/** Phantom marker carrying `Tx` in a directly-inferable position, so the
|
|
80
|
+
* consume loops' `TTx` type parameter resolves from the `sink` option
|
|
81
|
+
* (never set at runtime). */
|
|
82
|
+
readonly _tx?: Tx;
|
|
83
|
+
/** Static capabilities, read by the loop before the first fetch to fail
|
|
84
|
+
* fast on impossible pairings. */
|
|
85
|
+
readonly capabilities?: {
|
|
86
|
+
/** This sink cannot undo committed rows (append-only store, ClickHouse,
|
|
87
|
+
* parquet, …), so `rollback` is unimplementable and following the
|
|
88
|
+
* unfinalized tip would corrupt it on the first fork. The loop throws
|
|
89
|
+
* loudly unless consuming with `finalizedOnly: true` — in that mode
|
|
90
|
+
* reorgs never reach the sink and `rollback` may simply throw. */
|
|
91
|
+
finalizedOnly?: boolean;
|
|
92
|
+
};
|
|
93
|
+
/** The committed checkpoint, or `null` on first run. Called once, before
|
|
94
|
+
* the first page. Implementations may create their checkpoint storage
|
|
95
|
+
* here and SHOULD validate their rollback preconditions (e.g. that every
|
|
96
|
+
* declared table carries the height column). */
|
|
97
|
+
loadCursor(): Promise<string | null>;
|
|
98
|
+
/**
|
|
99
|
+
* Apply one batch atomically: open a transaction, run `write(tx)` (the
|
|
100
|
+
* user's inserts), and commit the rows AND `cursor` together. A throw
|
|
101
|
+
* from `write` must abort the whole transaction — leaving neither rows
|
|
102
|
+
* nor cursor, so a crashed batch is simply re-read on restart.
|
|
103
|
+
*/
|
|
104
|
+
commitBatch(cursor: string, write: (tx: Tx) => Promise<void> | void): Promise<void>;
|
|
105
|
+
/**
|
|
106
|
+
* Roll the projection back to the fork: delete everything AT OR ABOVE
|
|
107
|
+
* `forkPointHeight` (inclusive `>=` — the new chain re-supplies the fork
|
|
108
|
+
* block) and commit `rewindCursor` in the SAME transaction. Deleting
|
|
109
|
+
* without the rewound cursor is the classic silent-gap bug: a crash
|
|
110
|
+
* between the two writes resumes above the fork and the deleted range is
|
|
111
|
+
* never re-read.
|
|
112
|
+
*
|
|
113
|
+
* On a multi-fork page this is called once PER fork, every call with the
|
|
114
|
+
* same `rewindCursor` (the lowest fork point) but its own
|
|
115
|
+
* `forkPointHeight` — scope the undo by `forkPointHeight` only (contract
|
|
116
|
+
* invariant #11), and expect re-application after a crash (#10).
|
|
117
|
+
*/
|
|
118
|
+
rollback(forkPointHeight: number, rewindCursor: string): Promise<void>;
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* The sink conformance kit: executable probes for the {@link ConsumerSink}
|
|
122
|
+
* contract invariants — the ones that are SILENT when violated in
|
|
123
|
+
* production (torn batches, silent gaps, under-deleted forks) and only
|
|
124
|
+
* surface weeks later as missing or duplicated rows.
|
|
125
|
+
*
|
|
126
|
+
* Run every sink you author through it, in CI, against the real store:
|
|
127
|
+
*
|
|
128
|
+
* ```ts
|
|
129
|
+
* import { test } from "bun:test";
|
|
130
|
+
* import { attachSinkConformance } from "@secondlayer/sdk/sinks/testing";
|
|
131
|
+
*
|
|
132
|
+
* attachSinkConformance(test, {
|
|
133
|
+
* makeSink: () => mySink(db, { id: "conformance", tables: ["rows"], height: "height" }),
|
|
134
|
+
* reset: () => db.deleteFrom(...)...,
|
|
135
|
+
* insertRow: (tx, height, key) => ..., // MUST be replay-safe (upsert on key)
|
|
136
|
+
* readRows: () => ...,
|
|
137
|
+
* readCursor: () => ...,
|
|
138
|
+
* });
|
|
139
|
+
* ```
|
|
140
|
+
*
|
|
141
|
+
* Framework-agnostic: probes throw plain `Error`s, so they run under
|
|
142
|
+
* bun:test, vitest, jest, or a bare loop.
|
|
143
|
+
*/
|
|
144
|
+
interface SinkConformanceHarness<Tx = unknown> {
|
|
145
|
+
/** A fresh sink instance bound to the SAME store and the SAME checkpoint
|
|
146
|
+
* id each call — "new instance" is how the kit simulates a restart. */
|
|
147
|
+
makeSink(): ConsumerSink<Tx> | Promise<ConsumerSink<Tx>>;
|
|
148
|
+
/** Wipe the store back to empty: projection rows AND this id's
|
|
149
|
+
* checkpoint. Called before every probe. */
|
|
150
|
+
reset(): Promise<void> | void;
|
|
151
|
+
/** Insert one projection row through the lent transaction, exactly as a
|
|
152
|
+
* real `onBatch` handler would. MUST be replay-safe (upsert / insert-
|
|
153
|
+
* on-conflict-do-nothing keyed on `key`) — the same requirement the
|
|
154
|
+
* contract's at-least-once delivery puts on real handlers. */
|
|
155
|
+
insertRow(tx: Tx, height: number, key: string): Promise<void> | void;
|
|
156
|
+
/** Every committed projection row, read OUTSIDE any transaction. */
|
|
157
|
+
readRows(): Promise<Array<{
|
|
158
|
+
height: number;
|
|
159
|
+
key: string;
|
|
160
|
+
}>>;
|
|
161
|
+
/** The committed checkpoint cursor as visible to a restarted process, or
|
|
162
|
+
* `null` when none was ever committed. */
|
|
163
|
+
readCursor(): Promise<string | null>;
|
|
164
|
+
/** Optional (sinks with a writer lock): hold the lock for this sink's id
|
|
165
|
+
* — as a second live consumer would — while `during` runs. Enables the
|
|
166
|
+
* lock-contention probe (invariant #13: fail loudly, never interleave). */
|
|
167
|
+
withLockHeld?(during: () => Promise<void>): Promise<void>;
|
|
168
|
+
}
|
|
169
|
+
/** One executable contract probe; `run` throws on violation. */
|
|
170
|
+
interface ConformanceProbe {
|
|
171
|
+
name: string;
|
|
172
|
+
run(): Promise<void>;
|
|
173
|
+
}
|
|
174
|
+
/** Build the probe list for a harness. Prefer {@link attachSinkConformance}
|
|
175
|
+
* inside a test framework; use this directly for a bare runner. */
|
|
176
|
+
declare function sinkConformanceProbes<Tx>(harness: SinkConformanceHarness<Tx>): ConformanceProbe[];
|
|
177
|
+
/**
|
|
178
|
+
* Register every conformance probe with a test framework: pass your
|
|
179
|
+
* framework's `test` function (bun:test, vitest, jest — anything with a
|
|
180
|
+
* `(name, fn)` signature). One probe becomes one test case.
|
|
181
|
+
*/
|
|
182
|
+
declare function attachSinkConformance<Tx>(test: (name: string, fn: () => Promise<void>) => void, harness: SinkConformanceHarness<Tx>): void;
|
|
183
|
+
export { sinkConformanceProbes, attachSinkConformance, SinkConformanceHarness, ConformanceProbe };
|