@zackbart/connecta 0.37.1 → 0.38.0

package/dist/storage/d1-copy.js

@@ -1,255 +0,0 @@
-// Native copy orchestration. Only Workers KV and Web Crypto are Promise leaves.
-import { Effect } from "effect";
-import { errorLabel } from "../operator-record.js";
-import { KvToD1CopyError } from "../d1.js";
-import { familyOfKey, KEY_FAMILIES, KV_COPY_CURSOR_TTL_SECONDS, kvCopyKeys, kvCutoverKeys } from "./keys.js";
-import { copyIntoSqlEffect, nativeSqlStorage } from "./sql-native.js";
-import { nativeSqlDriver } from "./sql-driver.js";
-import { d1Driver } from "./d1-driver.js";
-const leaf = (operation) => Effect.tryPromise({ try: operation, catch: (error) => error });
-// A native equivalent of a Promise try/catch. Synchronous parser/validation
-// throws keep their original identity too; interruption is never recovered.
-const attempt = (effect) => Effect.result(effect.pipe(Effect.catchDefect((error) => Effect.fail(error))));
-/** D1 string and row limit, in UTF-8 bytes.
- * https://developers.cloudflare.com/d1/platform/limits/
- */
-const D1_MAX_ROW_BYTES = 2000000;
-/** Buffered entries, excluding the single value currently being read. */
-const COPY_BUFFER_BYTES = 4 * 1024 * 1024;
-const COPY_GROUP_KEYS = 50;
-const KV_LIST_LIMIT = 1000;
-const encoder = new TextEncoder();
-// SQLite record header, two TEXT serial types, and an eight-byte expiry.
-// Reserve the worst-case header rather than accepting a row at the boundary.
-const ROW_OVERHEAD_BYTES = 32;
-const emptyCounts = () => ({
-    copied: 0,
-    unchanged: 0,
-    conflicts: 0,
-    overwritten: 0,
-    expired: 0,
-    invalid: 0,
-    verified: 0,
-    mismatches: 0,
-});
-const OUTCOME_COUNT = {
-    copied: "copied",
-    unchanged: "unchanged",
-    conflict: "conflicts",
-    overwritten: "overwritten",
-};
-function errorClass(error) {
-    const label = errorLabel(error);
-    return label ? ` (${label})` : "";
-}
-const valueHash = (value) => Effect.gen(function* () {
-    const hash = yield* leaf(() => crypto.subtle.digest("SHA-256", encoder.encode(value)));
-    return Array.from(new Uint8Array(hash), (byte) => byte.toString(16).padStart(2, "0")).join("");
-});
-/** Mark cutover before reopening traffic. The marker stays in D1 permanently. */
-export function markKvToD1LiveEffect(db, source) {
-    return Effect.gen(function* () {
-        if (typeof source !== "string" || !source || source.includes("\0")) {
-            throw new TypeError("copyKvToD1 source must be a namespace id");
-        }
-        const cutover = yield* attempt(nativeSqlStorage(nativeSqlDriver(d1Driver(db))).set(kvCutoverKeys.source(source), "live"));
-        if (cutover._tag === "Failure") {
-            const error = cutover.failure;
-            throw new KvToD1CopyError(`Workers KV to D1 copy stopped: recording cutover failed${errorClass(error)}`, undefined);
-        }
-    });
-}
-/**
- * Copy live KV entries verbatim while all writers and traffic are stopped.
- * A live D1 row is kept unless its family is explicitly allowed to overwrite.
- * `verify` reads hashes and absolute expiries without changing copied entries.
- * Results contain counts only. Resume tokens are source-bound atomic claims.
- */
-export function copyKvToD1Effect(kv, db, options) {
-    return Effect.gen(function* () {
-        if (!options?.source || typeof options.source !== "string" || options.source.includes("\0")) {
-            throw new TypeError("copyKvToD1 source must be a namespace id");
-        }
-        const maxKeys = options.maxKeys ?? 500;
-        if (!Number.isSafeInteger(maxKeys) || maxKeys < 1) {
-            throw new TypeError("copyKvToD1 maxKeys must be a positive integer");
-        }
-        const allowed = new Set([...KEY_FAMILIES.map((family) => family.name), "connector-owned", "unclassified"]);
-        if (options.overwriteFamilies?.some((family) => !allowed.has(family))) {
-            throw new TypeError("copyKvToD1 overwriteFamilies contains an unknown family");
-        }
-        const overwrite = new Set(options.overwriteFamilies ?? []);
-        const driver = nativeSqlDriver(d1Driver(db));
-        const storage = nativeSqlStorage(driver);
-        const remember = (kvCursor) => Effect.gen(function* () {
-            const token = crypto.randomUUID();
-            yield* storage.set(kvCopyKeys.cursor(token), JSON.stringify({ source: options.source, cursor: kvCursor }), {
-                ttlSeconds: KV_COPY_CURSOR_TTL_SECONDS,
-            });
-            return token;
-        });
-        let cursor;
-        let pageCursor;
-        const families = {};
-        const count = (key, field) => {
-            (families[familyOfKey(key)] ??= emptyCounts())[field] += 1;
-        };
-        const unknown = new TypeError("copyKvToD1 cursor is unknown, expired, spent, or belongs to another source; restart under maintenance");
-        const stale = new TypeError("copyKvToD1 refuses stale KV after cutover");
-        // Validate before any D1 access so a failing binding cannot return raw input
-        // as an error's resume point.
-        if (options.cursor !== undefined &&
-            !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(options.cursor)) {
-            throw unknown;
-        }
-        const failures = new WeakSet();
-        const failure = (message) => {
-            const error = new KvToD1CopyError(message, undefined);
-            failures.add(error);
-            return error;
-        };
-        const claim = yield* attempt(Effect.gen(function* () {
-            if (!options.allowStale && (yield* storage.get(kvCutoverKeys.source(options.source))) !== null) {
-                throw stale;
-            }
-            if (options.cursor !== undefined) {
-                const key = kvCopyKeys.cursor(options.cursor);
-                const stored = yield* storage.get(key);
-                let record = null;
-                try {
-                    record = stored === null ? null : JSON.parse(stored);
-                }
-                catch {
-                    /* refused below */
-                }
-                if (!record || record.source !== options.source || typeof record.cursor !== "string") {
-                    throw unknown;
-                }
-                // Delete only the exact record read: concurrent consumers cannot both win.
-                if (!(yield* storage.compareAndSet(key, stored, null)))
-                    throw unknown;
-                cursor = record.cursor;
-            }
-        }));
-        if (claim._tag === "Failure") {
-            const error = claim.failure;
-            if (error === unknown || error === stale)
-                throw error;
-            throw new KvToD1CopyError(`Workers KV to D1 copy stopped: claiming the resume point in D1 failed${errorClass(error)}`, options.cursor);
-        }
-        const flush = (buffer) => Effect.gen(function* () {
-            const entries = buffer.filter((entry) => {
-                if (entry.expiresAtMs !== null && entry.expiresAtMs <= Date.now()) {
-                    count(entry.key, "expired");
-                    return false;
-                }
-                return true;
-            });
-            if (!entries.length)
-                return;
-            const flushed = yield* attempt(Effect.gen(function* () {
-                if (options.verify) {
-                    for (const entry of entries) {
-                        const [row] = yield* driver.all({
-                            // Read bytes, including NUL, and hash in JS. D1 has no SHA-256 function.
-                            sql: "SELECT CAST(value AS BLOB) AS value_bytes, expires_at_ms FROM connecta_kv WHERE key = ?",
-                            params: [entry.key],
-                        });
-                        const bytes = row ? Uint8Array.from(row.value_bytes) : undefined;
-                        const hash = bytes ? yield* leaf(() => crypto.subtle.digest("SHA-256", bytes)) : undefined;
-                        const hex = hash
-                            ? Array.from(new Uint8Array(hash), (byte) => byte.toString(16).padStart(2, "0")).join("")
-                            : undefined;
-                        count(entry.key, row && row.expires_at_ms === entry.expiresAtMs && hex === (yield* valueHash(entry.value))
-                            ? "verified"
-                            : "mismatches");
-                    }
-                }
-                else {
-                    // Partition only this bounded buffer by overwrite permission.
-                    for (const replace of [false, true]) {
-                        const group = entries.filter((entry) => overwrite.has(familyOfKey(entry.key)) === replace);
-                        if (!group.length)
-                            continue;
-                        const outcomes = yield* copyIntoSqlEffect(driver, group, { overwrite: replace, now: Date.now() });
-                        for (const [index, outcome] of outcomes.entries())
-                            count(group[index].key, OUTCOME_COUNT[outcome]);
-                    }
-                }
-            }));
-            if (flushed._tag === "Failure") {
-                const error = flushed.failure;
-                throw failure(`Workers KV to D1 copy stopped: ${options.verify ? "verifying" : "writing to"} D1 failed${errorClass(error)}`);
-            }
-        });
-        let read = 0;
-        const copied = yield* attempt(Effect.gen(function* () {
-            for (;;) {
-                pageCursor = cursor;
-                const listed = yield* attempt(leaf(() => kv.list({ limit: Math.min(KV_LIST_LIMIT, maxKeys - read), ...(cursor ? { cursor } : {}) })));
-                if (listed._tag === "Failure") {
-                    const error = listed.failure;
-                    throw failure(`Workers KV to D1 copy stopped: listing Workers KV failed${errorClass(error)}`);
-                }
-                const page = listed.success;
-                read += page.keys.length;
-                let entries = [];
-                let bufferedBytes = 0;
-                for (const key of page.keys) {
-                    const keyBytes = encoder.encode(key.name).byteLength;
-                    if (key.name.includes("\0") || keyBytes + ROW_OVERHEAD_BYTES > D1_MAX_ROW_BYTES) {
-                        count(key.name, "invalid");
-                        continue;
-                    }
-                    const expiresAtMs = key.expiration === undefined ? null : key.expiration * 1000;
-                    if (expiresAtMs !== null && expiresAtMs <= Date.now()) {
-                        count(key.name, "expired");
-                        continue;
-                    }
-                    const readValue = yield* attempt(leaf(() => kv.get(key.name, "text")));
-                    if (readValue._tag === "Failure") {
-                        const error = readValue.failure;
-                        throw failure(`Workers KV to D1 copy stopped: reading an entry of the ${familyOfKey(key.name)} family from Workers KV failed${errorClass(error)}`);
-                    }
-                    const value = readValue.success;
-                    if (value === null || (expiresAtMs !== null && expiresAtMs <= Date.now())) {
-                        count(key.name, "expired");
-                        continue;
-                    }
-                    const valueBytes = encoder.encode(value).byteLength;
-                    const bytes = keyBytes + valueBytes + ROW_OVERHEAD_BYTES;
-                    if (valueBytes > D1_MAX_ROW_BYTES || bytes > D1_MAX_ROW_BYTES) {
-                        count(key.name, "invalid");
-                        continue;
-                    }
-                    if (entries.length >= COPY_GROUP_KEYS || bufferedBytes + bytes > COPY_BUFFER_BYTES) {
-                        yield* flush(entries);
-                        entries = [];
-                        bufferedBytes = 0;
-                    }
-                    entries.push({ key: key.name, value, expiresAtMs });
-                    bufferedBytes += bytes;
-                }
-                yield* flush(entries);
-                if (page.list_complete || !page.cursor)
-                    return { done: true, families };
-                cursor = page.cursor;
-                if (read >= maxKeys)
-                    return { done: false, cursor: yield* remember(cursor), families };
-            }
-        }));
-        if (copied._tag === "Failure") {
-            const error = copied.failure;
-            // The old token was claimed atomically. Replace it on failure, if D1 works.
-            const resume = pageCursor === undefined
-                ? undefined
-                : yield* attempt(remember(pageCursor)).pipe(Effect.map((result) => (result._tag === "Success" ? result.success : undefined)));
-            throw new KvToD1CopyError(error instanceof KvToD1CopyError && failures.has(error)
-                ? error.message
-                : `Workers KV to D1 copy stopped: recording the resume point in D1 failed${errorClass(error)}`, resume);
-        }
-        else {
-            return copied.success;
-        }
-    });
-}

package/dist/storage/sql-schema.d.ts

@@ -1,6 +0,0 @@
-/**
- * The key-value table. Its shape is the one the 0.28 Worker example's
- * `d1-storage.ts` documented, so a D1 database that already holds it is read
- * as is.
- */
-export declare const KV_SCHEMA: readonly string[];

package/examples/worker/kv-to-d1.wrangler.jsonc

@@ -1,26 +0,0 @@
-// Bindings for the one-shot Workers KV → D1 copy in scripts/copy-kv-to-d1.mjs.
-// Never deployed: `wrangler deploy` reads wrangler.jsonc, which binds no KV.
-// Both bindings are remote, so the script reads the real namespace and writes
-// the real database through your `wrangler login` session. Fill in the ids,
-// stop traffic/writers, wait for stable KV, copy and verify before deploying,
-// then mark cutover before reopening traffic. README.md § "Upgrading from
-// 0.28" has the steps.
-{
-  "name": "connecta-kv-to-d1",
-  "compatibility_date": "2025-01-01",
-  "kv_namespaces": [
-    {
-      "binding": "CONNECTA_KV",
-      "id": "replace-with-kv-namespace-id",
-      "remote": true,
-    },
-  ],
-  "d1_databases": [
-    {
-      "binding": "CONNECTA_DB",
-      "database_name": "connecta",
-      "database_id": "replace-with-d1-database-id",
-      "remote": true,
-    },
-  ],
-}

package/examples/worker/scripts/copy-kv-to-d1.mjs

@@ -1,162 +0,0 @@
-#!/usr/bin/env node
-// Run only during the maintenance-window cutover. See README "Upgrading from 0.28".
-
-export const HELP = `Usage: node scripts/copy-kv-to-d1.mjs [config] --maintenance [options]
-Config defaults to kv-to-d1.wrangler.jsonc; source is its CONNECTA_KV namespace id.
-1. Bulk-export KV and export D1 with wrangler d1 export. Preserve the credentials key.
-2. Block traffic and all background writers, then drain in-flight requests.
-3. Wait at least 60 seconds, or the longest configured KV cacheTtl, whichever is
-   greater. Propagation can take longer. Two consecutive listing/hash passes
-   must match; this script checks them before copying or verifying.
-4. Copy BEFORE switching traffic. Resolve invalid/conflict counts under maintenance.
-   --overwrite-family <family> may be repeated; there is no blanket overwrite.
-   Token/OAuth/vault families also require --confirm-stale-d1 after the operator
-   confirms that D1's conflicting rows are stale. Run --verify for exact value
-   hashes and expiries; it reports mismatch counts by family, never payloads.
-5. Deploy D1 config under maintenance. Run doctor and check an existing OAuth
-   connector, cta_ token, vault credential, and visible activity. Set
-   CONNECTA_ACTIVITY="on"; config lives in src/connecta.config.ts.
-6. Run --mark-live before reopening traffic. It permanently seals this source
-   in D1. Never rerun afterward. --i-know-this-is-stale overrides the seal only
-   after assessing stale KV state, with traffic and writers stopped again.
-   Keep backups before deleting KV.
-Options: --verify, --mark-live, --overwrite-family <family>, --confirm-stale-d1,
-         --i-know-this-is-stale, --maintenance, --help`;
-
-export class MigrationUsageError extends Error {}
-
-export function parseArgs(args) {
-  const options = { configPath: "kv-to-d1.wrangler.jsonc", overwriteFamilies: [] };
-  let hasConfig = false;
-  for (let i = 0; i < args.length; i++) {
-    const arg = args[i];
-    if (arg === "--overwrite-family") {
-      const family = args[++i];
-      if (!family || family.startsWith("--")) throw new MigrationUsageError("--overwrite-family requires a family");
-      options.overwriteFamilies.push(family);
-    } else if (arg === "--maintenance") options.maintenance = true;
-    else if (arg === "--verify") options.verify = true;
-    else if (arg === "--mark-live") options.markLive = true;
-    else if (arg === "--confirm-stale-d1") options.confirmStale = true;
-    else if (arg === "--i-know-this-is-stale") options.allowStale = true;
-    else if (arg === "--help") options.help = true;
-    else if (!arg.startsWith("--") && !hasConfig) {
-      options.configPath = arg;
-      hasConfig = true;
-    } else throw new MigrationUsageError("Unknown argument; use --help");
-  }
-  if (options.help) return options;
-  if (!options.maintenance)
-    throw new MigrationUsageError(
-      "Traffic and writers must be stopped; pass --maintenance after draining and waiting for KV propagation",
-    );
-  if (
-    options.overwriteFamilies.some(
-      (family) => family === "access-token" || family === "credential" || family.startsWith("oauth"),
-    ) &&
-    !options.confirmStale
-  ) {
-    throw new MigrationUsageError(
-      "Token/OAuth/vault overwrite requires --confirm-stale-d1 after confirming D1 rows are stale",
-    );
-  }
-  if ((options.verify || options.markLive) && options.overwriteFamilies.length)
-    throw new MigrationUsageError("Verification and cutover marking cannot overwrite families");
-  if (options.verify && options.markLive)
-    throw new MigrationUsageError("Verify before marking cutover in a separate command");
-  return options;
-}
-
-const encode = new TextEncoder();
-const hash = async (value) => {
-  const bytes = await crypto.subtle.digest("SHA-256", encode.encode(value));
-  return Array.from(new Uint8Array(bytes), (n) => n.toString(16).padStart(2, "0")).join("");
-};
-
-// Keep only entry digests. Sorting makes the comparison independent of list order.
-async function sourceHash(kv) {
-  const digests = [];
-  let cursor;
-  do {
-    const page = await kv.list({ limit: 1000, ...(cursor ? { cursor } : {}) });
-    for (const key of page.keys) {
-      if (key.expiration !== undefined && key.expiration * 1000 <= Date.now()) continue;
-      const value = await kv.get(key.name, "text");
-      if (value !== null) digests.push(await hash(JSON.stringify([key.name, key.expiration ?? null, value])));
-    }
-    cursor = page.list_complete ? undefined : page.cursor;
-  } while (cursor);
-  return hash(digests.sort().join(""));
-}
-
-/** Injected operations let the Workers suite exercise the real migration flow. */
-export async function runMigration(kv, db, options, operations, output = console) {
-  if (!options.maintenance) throw new MigrationUsageError("Migration requires maintenance");
-  if (options.markLive) {
-    await operations.markKvToD1Live(db, options.source);
-    output.log("Cutover sealed in D1. Reopen traffic after deployment verification; never copy stale KV afterward.");
-    return 0;
-  }
-  if ((await sourceHash(kv)) !== (await sourceHash(kv))) {
-    throw new MigrationUsageError("Workers KV source is not stable; keep maintenance active, wait, and repeat");
-  }
-  const families = {};
-  let cursor;
-  do {
-    const result = await operations.copyKvToD1(kv, db, {
-      source: options.source,
-      cursor,
-      overwriteFamilies: options.overwriteFamilies,
-      verify: options.verify,
-      allowStale: options.allowStale,
-    });
-    for (const [family, counts] of Object.entries(result.families)) {
-      const into = (families[family] ??= {});
-      for (const [field, n] of Object.entries(counts)) into[field] = (into[field] ?? 0) + n;
-    }
-    cursor = result.cursor;
-  } while (cursor);
-  output.table(families);
-  if (Object.values(families).some((counts) => counts.invalid > 0 || counts.conflicts > 0 || counts.mismatches > 0)) {
-    output.error(
-      "Invalid entries, conflicts, or verification mismatches remain. Keep maintenance active and resolve the per-family counts.",
-    );
-    return 1;
-  }
-  return 0;
-}
-
-async function main() {
-  let proxy;
-  let operations;
-  try {
-    const options = parseArgs(process.argv.slice(2));
-    if (options.help) {
-      console.log(HELP);
-      return;
-    }
-    const wranglerModule = "wrangler";
-    const d1Module = "@zackbart/connecta/d1";
-    const { getPlatformProxy, unstable_readConfig } = await import(/* @vite-ignore */ wranglerModule);
-    operations = await import(/* @vite-ignore */ d1Module);
-    const config = unstable_readConfig({ config: options.configPath });
-    const source = config.kv_namespaces?.find((binding) => binding.binding === "CONNECTA_KV")?.id;
-    if (!source || source.startsWith("replace-"))
-      throw new MigrationUsageError("Configure the CONNECTA_KV namespace id before copying");
-    proxy = await getPlatformProxy({ configPath: options.configPath, persist: false });
-    const { CONNECTA_KV: kv, CONNECTA_DB: db } = proxy.env;
-    if (!kv || !db) throw new MigrationUsageError("Copy config must bind CONNECTA_KV and CONNECTA_DB");
-    process.exitCode = await runMigration(kv, db, { ...options, source }, operations);
-  } catch (error) {
-    console.error(
-      error instanceof MigrationUsageError || (operations && error instanceof operations.KvToD1CopyError)
-        ? error.message
-        : "Workers KV to D1 migration stopped; keep maintenance active and inspect the configuration",
-    );
-    process.exitCode = 1;
-  } finally {
-    await proxy?.dispose();
-  }
-}
-
-if (typeof process !== "undefined" && import.meta.url === `file://${process.argv?.[1]}`) await main();