@zackbart/connecta 0.34.1 → 0.35.0

package/dist/storage/d1-copy.js

@@ -0,0 +1,255 @@
+// Native copy orchestration. Only Workers KV and Web Crypto are Promise leaves.
+import { Effect } from "effect";
+import { errorLabel } from "../operator-record.js";
+import { KvToD1CopyError } from "../d1.js";
+import { familyOfKey, KEY_FAMILIES, KV_COPY_CURSOR_TTL_SECONDS, kvCopyKeys, kvCutoverKeys } from "./keys.js";
+import { copyIntoSqlEffect, nativeSqlStorage } from "./sql-native.js";
+import { nativeSqlDriver } from "./sql-driver.js";
+import { d1Driver } from "./d1-driver.js";
+const leaf = (operation) => Effect.tryPromise({ try: operation, catch: (error) => error });
+// A native equivalent of a Promise try/catch. Synchronous parser/validation
+// throws keep their original identity too; interruption is never recovered.
+const attempt = (effect) => Effect.result(effect.pipe(Effect.catchDefect((error) => Effect.fail(error))));
+/** D1 string and row limit, in UTF-8 bytes.
+ * https://developers.cloudflare.com/d1/platform/limits/
+ */
+const D1_MAX_ROW_BYTES = 2000000;
+/** Buffered entries, excluding the single value currently being read. */
+const COPY_BUFFER_BYTES = 4 * 1024 * 1024;
+const COPY_GROUP_KEYS = 50;
+const KV_LIST_LIMIT = 1000;
+const encoder = new TextEncoder();
+// SQLite record header, two TEXT serial types, and an eight-byte expiry.
+// Reserve the worst-case header rather than accepting a row at the boundary.
+const ROW_OVERHEAD_BYTES = 32;
+const emptyCounts = () => ({
+    copied: 0,
+    unchanged: 0,
+    conflicts: 0,
+    overwritten: 0,
+    expired: 0,
+    invalid: 0,
+    verified: 0,
+    mismatches: 0,
+});
+const OUTCOME_COUNT = {
+    copied: "copied",
+    unchanged: "unchanged",
+    conflict: "conflicts",
+    overwritten: "overwritten",
+};
+function errorClass(error) {
+    const label = errorLabel(error);
+    return label ? ` (${label})` : "";
+}
+const valueHash = (value) => Effect.gen(function* () {
+    const hash = yield* leaf(() => crypto.subtle.digest("SHA-256", encoder.encode(value)));
+    return Array.from(new Uint8Array(hash), (byte) => byte.toString(16).padStart(2, "0")).join("");
+});
+/** Mark cutover before reopening traffic. The marker stays in D1 permanently. */
+export function markKvToD1LiveEffect(db, source) {
+    return Effect.gen(function* () {
+        if (typeof source !== "string" || !source || source.includes("\0")) {
+            throw new TypeError("copyKvToD1 source must be a namespace id");
+        }
+        const cutover = yield* attempt(nativeSqlStorage(nativeSqlDriver(d1Driver(db))).set(kvCutoverKeys.source(source), "live"));
+        if (cutover._tag === "Failure") {
+            const error = cutover.failure;
+            throw new KvToD1CopyError(`Workers KV to D1 copy stopped: recording cutover failed${errorClass(error)}`, undefined);
+        }
+    });
+}
+/**
+ * Copy live KV entries verbatim while all writers and traffic are stopped.
+ * A live D1 row is kept unless its family is explicitly allowed to overwrite.
+ * `verify` reads hashes and absolute expiries without changing copied entries.
+ * Results contain counts only. Resume tokens are source-bound atomic claims.
+ */
+export function copyKvToD1Effect(kv, db, options) {
+    return Effect.gen(function* () {
+        if (!options?.source || typeof options.source !== "string" || options.source.includes("\0")) {
+            throw new TypeError("copyKvToD1 source must be a namespace id");
+        }
+        const maxKeys = options.maxKeys ?? 500;
+        if (!Number.isSafeInteger(maxKeys) || maxKeys < 1) {
+            throw new TypeError("copyKvToD1 maxKeys must be a positive integer");
+        }
+        const allowed = new Set([...KEY_FAMILIES.map((family) => family.name), "connector-owned", "unclassified"]);
+        if (options.overwriteFamilies?.some((family) => !allowed.has(family))) {
+            throw new TypeError("copyKvToD1 overwriteFamilies contains an unknown family");
+        }
+        const overwrite = new Set(options.overwriteFamilies ?? []);
+        const driver = nativeSqlDriver(d1Driver(db));
+        const storage = nativeSqlStorage(driver);
+        const remember = (kvCursor) => Effect.gen(function* () {
+            const token = crypto.randomUUID();
+            yield* storage.set(kvCopyKeys.cursor(token), JSON.stringify({ source: options.source, cursor: kvCursor }), {
+                ttlSeconds: KV_COPY_CURSOR_TTL_SECONDS,
+            });
+            return token;
+        });
+        let cursor;
+        let pageCursor;
+        const families = {};
+        const count = (key, field) => {
+            (families[familyOfKey(key)] ??= emptyCounts())[field] += 1;
+        };
+        const unknown = new TypeError("copyKvToD1 cursor is unknown, expired, spent, or belongs to another source; restart under maintenance");
+        const stale = new TypeError("copyKvToD1 refuses stale KV after cutover");
+        // Validate before any D1 access so a failing binding cannot return raw input
+        // as an error's resume point.
+        if (options.cursor !== undefined &&
+            !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(options.cursor)) {
+            throw unknown;
+        }
+        const failures = new WeakSet();
+        const failure = (message) => {
+            const error = new KvToD1CopyError(message, undefined);
+            failures.add(error);
+            return error;
+        };
+        const claim = yield* attempt(Effect.gen(function* () {
+            if (!options.allowStale && (yield* storage.get(kvCutoverKeys.source(options.source))) !== null) {
+                throw stale;
+            }
+            if (options.cursor !== undefined) {
+                const key = kvCopyKeys.cursor(options.cursor);
+                const stored = yield* storage.get(key);
+                let record = null;
+                try {
+                    record = stored === null ? null : JSON.parse(stored);
+                }
+                catch {
+                    /* refused below */
+                }
+                if (!record || record.source !== options.source || typeof record.cursor !== "string") {
+                    throw unknown;
+                }
+                // Delete only the exact record read: concurrent consumers cannot both win.
+                if (!(yield* storage.compareAndSet(key, stored, null)))
+                    throw unknown;
+                cursor = record.cursor;
+            }
+        }));
+        if (claim._tag === "Failure") {
+            const error = claim.failure;
+            if (error === unknown || error === stale)
+                throw error;
+            throw new KvToD1CopyError(`Workers KV to D1 copy stopped: claiming the resume point in D1 failed${errorClass(error)}`, options.cursor);
+        }
+        const flush = (buffer) => Effect.gen(function* () {
+            const entries = buffer.filter((entry) => {
+                if (entry.expiresAtMs !== null && entry.expiresAtMs <= Date.now()) {
+                    count(entry.key, "expired");
+                    return false;
+                }
+                return true;
+            });
+            if (!entries.length)
+                return;
+            const flushed = yield* attempt(Effect.gen(function* () {
+                if (options.verify) {
+                    for (const entry of entries) {
+                        const [row] = yield* driver.all({
+                            // Read bytes, including NUL, and hash in JS. D1 has no SHA-256 function.
+                            sql: "SELECT CAST(value AS BLOB) AS value_bytes, expires_at_ms FROM connecta_kv WHERE key = ?",
+                            params: [entry.key],
+                        });
+                        const bytes = row ? Uint8Array.from(row.value_bytes) : undefined;
+                        const hash = bytes ? yield* leaf(() => crypto.subtle.digest("SHA-256", bytes)) : undefined;
+                        const hex = hash
+                            ? Array.from(new Uint8Array(hash), (byte) => byte.toString(16).padStart(2, "0")).join("")
+                            : undefined;
+                        count(entry.key, row && row.expires_at_ms === entry.expiresAtMs && hex === (yield* valueHash(entry.value))
+                            ? "verified"
+                            : "mismatches");
+                    }
+                }
+                else {
+                    // Partition only this bounded buffer by overwrite permission.
+                    for (const replace of [false, true]) {
+                        const group = entries.filter((entry) => overwrite.has(familyOfKey(entry.key)) === replace);
+                        if (!group.length)
+                            continue;
+                        const outcomes = yield* copyIntoSqlEffect(driver, group, { overwrite: replace, now: Date.now() });
+                        for (const [index, outcome] of outcomes.entries())
+                            count(group[index].key, OUTCOME_COUNT[outcome]);
+                    }
+                }
+            }));
+            if (flushed._tag === "Failure") {
+                const error = flushed.failure;
+                throw failure(`Workers KV to D1 copy stopped: ${options.verify ? "verifying" : "writing to"} D1 failed${errorClass(error)}`);
+            }
+        });
+        let read = 0;
+        const copied = yield* attempt(Effect.gen(function* () {
+            for (;;) {
+                pageCursor = cursor;
+                const listed = yield* attempt(leaf(() => kv.list({ limit: Math.min(KV_LIST_LIMIT, maxKeys - read), ...(cursor ? { cursor } : {}) })));
+                if (listed._tag === "Failure") {
+                    const error = listed.failure;
+                    throw failure(`Workers KV to D1 copy stopped: listing Workers KV failed${errorClass(error)}`);
+                }
+                const page = listed.success;
+                read += page.keys.length;
+                let entries = [];
+                let bufferedBytes = 0;
+                for (const key of page.keys) {
+                    const keyBytes = encoder.encode(key.name).byteLength;
+                    if (key.name.includes("\0") || keyBytes + ROW_OVERHEAD_BYTES > D1_MAX_ROW_BYTES) {
+                        count(key.name, "invalid");
+                        continue;
+                    }
+                    const expiresAtMs = key.expiration === undefined ? null : key.expiration * 1000;
+                    if (expiresAtMs !== null && expiresAtMs <= Date.now()) {
+                        count(key.name, "expired");
+                        continue;
+                    }
+                    const readValue = yield* attempt(leaf(() => kv.get(key.name, "text")));
+                    if (readValue._tag === "Failure") {
+                        const error = readValue.failure;
+                        throw failure(`Workers KV to D1 copy stopped: reading an entry of the ${familyOfKey(key.name)} family from Workers KV failed${errorClass(error)}`);
+                    }
+                    const value = readValue.success;
+                    if (value === null || (expiresAtMs !== null && expiresAtMs <= Date.now())) {
+                        count(key.name, "expired");
+                        continue;
+                    }
+                    const valueBytes = encoder.encode(value).byteLength;
+                    const bytes = keyBytes + valueBytes + ROW_OVERHEAD_BYTES;
+                    if (valueBytes > D1_MAX_ROW_BYTES || bytes > D1_MAX_ROW_BYTES) {
+                        count(key.name, "invalid");
+                        continue;
+                    }
+                    if (entries.length >= COPY_GROUP_KEYS || bufferedBytes + bytes > COPY_BUFFER_BYTES) {
+                        yield* flush(entries);
+                        entries = [];
+                        bufferedBytes = 0;
+                    }
+                    entries.push({ key: key.name, value, expiresAtMs });
+                    bufferedBytes += bytes;
+                }
+                yield* flush(entries);
+                if (page.list_complete || !page.cursor)
+                    return { done: true, families };
+                cursor = page.cursor;
+                if (read >= maxKeys)
+                    return { done: false, cursor: yield* remember(cursor), families };
+            }
+        }));
+        if (copied._tag === "Failure") {
+            const error = copied.failure;
+            // The old token was claimed atomically. Replace it on failure, if D1 works.
+            const resume = pageCursor === undefined
+                ? undefined
+                : yield* attempt(remember(pageCursor)).pipe(Effect.map((result) => (result._tag === "Success" ? result.success : undefined)));
+            throw new KvToD1CopyError(error instanceof KvToD1CopyError && failures.has(error)
+                ? error.message
+                : `Workers KV to D1 copy stopped: recording the resume point in D1 failed${errorClass(error)}`, resume);
+        }
+        else {
+            return copied.success;
+        }
+    });
+}

package/dist/storage/d1-driver.js

@@ -0,0 +1,15 @@
+import { effectSqlDriver, sqlDriverEdge, sqlPromise } from "./sql-driver.js";
+export function d1Driver(db) {
+    const prepare = (statement) => statement.params.length > 0 ? db.prepare(statement.sql).bind(...statement.params) : db.prepare(statement.sql);
+    return sqlDriverEdge(effectSqlDriver({
+        all: (statement) => sqlPromise(() => prepare(statement)
+            .all()
+            .then((result) => result.results)),
+        run: (statement) => sqlPromise(() => prepare(statement)
+            .run()
+            .then((result) => result.meta.changes ?? 0)),
+        // D1's public binding API preserves per-statement changes, including zero;
+        // execute the whole array once. No withTransaction, RETURNING, or replay.
+        batch: (statements) => sqlPromise(() => db.batch(statements.map(prepare)).then((results) => results.map((result) => result.meta.changes ?? 0))),
+    }));
+}

package/dist/storage/keys.js

@@ -74,6 +74,19 @@ export const resultKeys = {
     chunk: (id, index) => validateStorageKey(index === 0 ? `result:${id}` : `result:${id}#${index}`),
 };
 // --- root families --------------------------------------------------------
+/** A deployment's random HMAC key for bound continuation cursors; never expires or leaves storage. */
+export const continuationKeys = {
+    family: {
+        name: "continuation-key",
+        scope: "root",
+        prefixes: ["continuation-key:v1:"],
+        version: { number: 1, in: "key" },
+        codec: textCodec,
+        ttl: { kind: "durable" },
+        durable: true,
+    },
+    signingKey: "continuation-key:v1:signing",
+};
 /**
  * The runtime-wide result stash ledger: every live stash entry's reservation
  * id, byte charge, and expiry. One record, swapped by compare-and-set, so the
@@ -188,6 +201,19 @@ export const accessTokenKeys = {
     adopted: (legacyRecordId) => validateStorageKey(`access-token:v1:adopted:${legacyRecordId}`),
     active: "access-token:v1:active",
 };
+/** Latest successful agent endpoint admission; name and timestamp only. */
+export const agentObservationKeys = {
+    family: {
+        name: "agent-observation",
+        scope: "root",
+        prefixes: ["agent-observation:v1:"],
+        version: { number: 1, in: "key" },
+        codec: textCodec,
+        ttl: { kind: "durable" },
+        durable: true,
+    },
+    agent: (name) => validateStorageKey(`agent-observation:v1:${name}`),
+};
 /**
  * A vault credential, sealed: `credential:v1` inside its connector's
  * namespace, under the owner's principal partition when it is personal.
@@ -399,6 +425,7 @@ export const inputRetryKeys = {
  * namespace, and are not listed here.
  */
 export const KEY_FAMILIES = [
+    continuationKeys.family,
     resultKeys.family,
     stashLedgerKeys.family,
     stashExpiryKeys.family,
@@ -406,6 +433,7 @@ export const KEY_FAMILIES = [
     responseCacheKeys.family,
     oauthHandoffKeys.family,
     accessTokenKeys.family,
+    agentObservationKeys.family,
     credentialKeys.family,
     oauthGrantKeys.family,
     oauthFlowKeys.family,

package/dist/storage/sql-driver.js

@@ -0,0 +1,88 @@
+// Native SQL service, private to the /d1 and /sqlite import graphs.
+// SqlClient services and Statement operations execute reads and writes. Atomic
+// batches retain the public platform metadata and structural D1 compatibility.
+import { Effect, Stream } from "effect";
+import * as SqlClient from "effect/sql/SqlClient";
+import * as Reactivity from "effect/reactivity/Reactivity";
+import { SqlError, UnknownError } from "effect/sql/SqlError";
+import * as Statement from "effect/sql/Statement";
+import { runEdge } from "../runtime/run.js";
+// SQL errors carry no driver cause or parameters, even inside tracing. Recover
+// the original thrown object only at the private SQL boundary, as before.
+const causes = new WeakMap();
+function sqlError(cause) {
+    const error = new SqlError({ reason: new UnknownError({ cause: undefined, message: "SQL operation failed" }) });
+    causes.set(error, cause);
+    return error;
+}
+const original = (error) => (causes.has(error) ? causes.get(error) : error);
+export function sqlSync(operation) {
+    return Effect.try({ try: operation, catch: sqlError });
+}
+/** Exactly one SDK/platform Promise leaf; no retries or shared flights. */
+export function sqlPromise(operation) {
+    return Effect.tryPromise({ try: operation, catch: sqlError });
+}
+/**
+ * A request-local public SqlClient service executes actual Statement Effects.
+ * Native batches are deliberately separate: D1 cannot use withTransaction and
+ * SQLite must not yield between BEGIN IMMEDIATE and COMMIT. The deployment
+ * owns the database; the service owns no platform connection or cached I/O.
+ */
+export function effectSqlDriver(leaves) {
+    const input = (sql, params) => ({
+        sql,
+        params: params,
+    });
+    const execute = (sql, params, transform) => leaves.all(input(sql, params)).pipe(Effect.map((rows) => (transform ? transform(rows) : rows)));
+    const values = (sql, params) => leaves.all(input(sql, params)).pipe(Effect.map((rows) => rows.map((row) => Object.values(row))));
+    const connection = {
+        execute,
+        executeRaw: (sql, params) => leaves.run(input(sql, params)),
+        executeUnprepared: execute,
+        executeValues: values,
+        executeValuesUnprepared: values,
+        executeStream: () => Stream.fail(sqlError(new Error("SQL streaming is not supported"))),
+    };
+    // This Effect constructs a fresh client and its services inside every
+    // operation. No request-owned object, scope or promise is memoized here.
+    const client = SqlClient.make({
+        acquirer: Effect.succeed(connection),
+        borrower: (use) => use(connection),
+        transactionAcquirer: Effect.fail(sqlError(new Error("Use an atomic SQL batch"))),
+        compiler: Statement.makeCompilerSqlite(),
+        spanAttributes: [],
+    }).pipe(Effect.provideServiceEffect(Reactivity.Reactivity, Reactivity.make));
+    const protect = (effect) => 
+    // The yield precedes physical execution, never an atomic transaction.
+    // Statement execution creates SQL spans; disable tracing before it starts,
+    // and unwrap only after the private SQL operation has completed.
+    Effect.andThen(Effect.yieldNow, effect).pipe(Effect.withTracerEnabled(false), Effect.mapError(original), Effect.catchDefect((error) => Effect.fail(error)));
+    const operation = (use) => protect(SqlClient.SqlClient.use(use).pipe(Effect.provideServiceEffect(SqlClient.SqlClient, client)));
+    return {
+        all: (statement) => operation((sql) => Effect.map(sql.unsafe(statement.sql, statement.params), (rows) => rows)),
+        run: (statement) => operation((sql) => sql.unsafe(statement.sql, statement.params).raw),
+        // One native transaction returns exact per-statement metadata. This
+        // capability is not SqlClient.withTransaction, which D1 does not support.
+        batch: (statements) => protect(Effect.suspend(() => leaves.batch(statements))),
+    };
+}
+const natives = new WeakMap();
+/** Promise compatibility view, including private adapter characterization. */
+export function sqlDriverEdge(native) {
+    const driver = {
+        all: (statement) => runEdge(native.all(statement)),
+        run: (statement) => runEdge(native.run(statement)),
+        batch: (statements) => runEdge(native.batch(statements)),
+    };
+    natives.set(driver, native);
+    return driver;
+}
+/** Custom/private Promise drivers retain their original fallback behavior. */
+export function nativeSqlDriver(driver) {
+    return (natives.get(driver) ?? {
+        all: (statement) => Effect.tryPromise({ try: () => Promise.resolve(driver.all(statement)), catch: (e) => e }),
+        run: (statement) => Effect.tryPromise({ try: () => Promise.resolve(driver.run(statement)), catch: (e) => e }),
+        batch: (statements) => Effect.tryPromise({ try: () => Promise.resolve(driver.batch(statements)), catch: (e) => e }),
+    });
+}