@reddoorla/maintenance 0.51.0 → 0.53.0

package/dist/cli/bin.js

@@ -2333,6 +2333,366 @@ var init_orchestrate = __esm({
   }
 });
 
+// src/db/migrations.ts
+var MIGRATIONS;
+var init_migrations = __esm({
+  "src/db/migrations.ts"() {
+    "use strict";
+    MIGRATIONS = [
+      {
+        id: "0001_init",
+        sql: `
+      CREATE TABLE IF NOT EXISTS submissions (
+        id TEXT PRIMARY KEY,
+        submission_id INTEGER,
+        site_id TEXT NOT NULL,
+        form_type TEXT NOT NULL,
+        name TEXT NOT NULL,
+        email TEXT NOT NULL,
+        phone TEXT,
+        message TEXT,
+        extra_fields TEXT,
+        source_url TEXT,
+        utm TEXT,
+        submitted_at TEXT,
+        status TEXT NOT NULL DEFAULT 'new',
+        notify_status TEXT NOT NULL DEFAULT 'skipped',
+        resend_message_id TEXT
+      );
+      CREATE INDEX IF NOT EXISTS idx_submissions_site_submitted
+        ON submissions (site_id, submitted_at DESC);
+      CREATE INDEX IF NOT EXISTS idx_submissions_status
+        ON submissions (status);
+      CREATE TABLE IF NOT EXISTS spam_screenouts (
+        site_id TEXT NOT NULL,
+        date TEXT NOT NULL,
+        honeypot INTEGER NOT NULL DEFAULT 0,
+        too_fast INTEGER NOT NULL DEFAULT 0,
+        marked_spam INTEGER NOT NULL DEFAULT 0,
+        PRIMARY KEY (site_id, date)
+      );
+    `
+      }
+    ];
+  }
+});
+
+// src/db/migrate.ts
+var migrate_exports = {};
+__export(migrate_exports, {
+  runMigrations: () => runMigrations
+});
+async function runMigrations(client) {
+  await client.execute(
+    "CREATE TABLE IF NOT EXISTS _migrations (id TEXT PRIMARY KEY, applied_at TEXT NOT NULL)"
+  );
+  const existing = await client.execute("SELECT id FROM _migrations");
+  const applied = new Set(existing.rows.map((r) => String(r.id)));
+  const ran = [];
+  for (const m of MIGRATIONS) {
+    if (applied.has(m.id)) continue;
+    await client.executeMultiple(m.sql);
+    await client.execute({
+      sql: "INSERT INTO _migrations (id, applied_at) VALUES (?, ?)",
+      args: [m.id, (/* @__PURE__ */ new Date()).toISOString()]
+    });
+    ran.push(m.id);
+  }
+  return ran;
+}
+var init_migrate = __esm({
+  "src/db/migrate.ts"() {
+    "use strict";
+    init_migrations();
+  }
+});
+
+// src/db/client.ts
+var client_exports2 = {};
+__export(client_exports2, {
+  openDb: () => openDb,
+  readDbConfig: () => readDbConfig
+});
+import { createClient } from "@libsql/client";
+import { Kysely } from "kysely";
+import { LibsqlDialect } from "@libsql/kysely-libsql";
+function missing2(name) {
+  return Object.assign(
+    new Error(
+      `${name} not set. Export it in your shell or put it in ${defaultCredentialsPath()} as ${name}=...`
+    ),
+    { exitCode: 2 }
+  );
+}
+function readDbConfig() {
+  const url = process.env.TURSO_DATABASE_URL;
+  if (!url) throw missing2("TURSO_DATABASE_URL");
+  const authToken = process.env.TURSO_AUTH_TOKEN;
+  return authToken ? { url, authToken } : { url };
+}
+async function openDb(cfg) {
+  const clientConfig = cfg.authToken ? { url: cfg.url, authToken: cfg.authToken } : { url: cfg.url };
+  const client = createClient(clientConfig);
+  await runMigrations(client);
+  return new Kysely({ dialect: new LibsqlDialect({ client }) });
+}
+var init_client2 = __esm({
+  "src/db/client.ts"() {
+    "use strict";
+    init_credentials();
+    init_migrate();
+  }
+});
+
+// src/forms/types.ts
+var SUBMISSION_FORM_TYPES;
+var init_types = __esm({
+  "src/forms/types.ts"() {
+    "use strict";
+    SUBMISSION_FORM_TYPES = [
+      "contact",
+      "inquiry",
+      "newsletter",
+      "rsvp",
+      "reserve"
+    ];
+  }
+});
+
+// src/reports/submission-row.ts
+function toFormType(raw) {
+  if (raw && SUBMISSION_FORM_TYPES.includes(raw)) return raw;
+  if (raw)
+    console.warn(`[submissions] unknown Form type ${JSON.stringify(raw)} \u2014 treating as contact`);
+  return "contact";
+}
+function toStatus(raw) {
+  if (raw && SUBMISSION_STATUSES.includes(raw))
+    return raw;
+  return "new";
+}
+function toNotifyStatus(raw) {
+  if (raw && NOTIFY_STATUSES.includes(raw)) return raw;
+  return "skipped";
+}
+var SUBMISSION_STATUSES, NOTIFY_STATUSES;
+var init_submission_row = __esm({
+  "src/reports/submission-row.ts"() {
+    "use strict";
+    init_types();
+    SUBMISSION_STATUSES = ["new", "read", "archived", "spam"];
+    NOTIFY_STATUSES = ["sent", "failed", "skipped"];
+  }
+});
+
+// src/reports/airtable/submissions.ts
+function mapRow3(rec) {
+  const f = rec.fields;
+  const linkSites = f["Site"] ?? [];
+  return {
+    id: rec.id,
+    submissionId: typeof f["Submission ID"] === "number" ? f["Submission ID"] : null,
+    siteId: linkSites[0] ?? "",
+    formType: toFormType(f["Form type"]),
+    name: String(f["Name"] ?? ""),
+    email: String(f["Email"] ?? ""),
+    phone: f["Phone"] ?? null,
+    message: f["Message"] ?? null,
+    extraFields: f["Extra fields"] ?? null,
+    sourceUrl: f["Source URL"] ?? null,
+    utm: f["UTM"] ?? null,
+    submittedAt: f["Submitted at"] ?? null,
+    status: toStatus(f["Status"]),
+    notifyStatus: toNotifyStatus(f["Notify status"]),
+    resendMessageId: f["Resend message ID"] ?? null
+  };
+}
+var SUBMISSIONS_TABLE;
+var init_submissions = __esm({
+  "src/reports/airtable/submissions.ts"() {
+    "use strict";
+    init_reports();
+    init_submission_row();
+    SUBMISSIONS_TABLE = "Submissions";
+  }
+});
+
+// src/reports/airtable/screenouts.ts
+function num(v) {
+  return typeof v === "number" && Number.isFinite(v) ? v : 0;
+}
+function siteIdOf(fields) {
+  const link = fields["Site"];
+  return link?.[0] ?? "";
+}
+async function listScreenOutsSince(base, sinceDate) {
+  const out = /* @__PURE__ */ new Map();
+  await base(SCREENOUTS_TABLE).select({ filterByFormula: `{Date} >= ${JSON.stringify(sinceDate)}`, pageSize: 100 }).eachPage((records, fetchNextPage) => {
+    for (const rec of records) {
+      const f = rec.fields;
+      const date = typeof f["Date"] === "string" ? f["Date"] : "";
+      if (date < sinceDate) continue;
+      const siteId = siteIdOf(f);
+      if (!siteId) continue;
+      const cur = out.get(siteId) ?? { honeypot: 0, tooFast: 0, markedSpam: 0 };
+      cur.honeypot += num(f["Honeypot"]);
+      cur.tooFast += num(f["Too-fast"]);
+      cur.markedSpam += num(f["Marked spam"]);
+      out.set(siteId, cur);
+    }
+    fetchNextPage();
+  });
+  return out;
+}
+var SCREENOUTS_TABLE;
+var init_screenouts = __esm({
+  "src/reports/airtable/screenouts.ts"() {
+    "use strict";
+    SCREENOUTS_TABLE = "Spam Screenouts";
+  }
+});
+
+// src/db/submissions.ts
+import { sql } from "kysely";
+async function backfillSubmission(db, row) {
+  await db.insertInto("submissions").values({
+    id: row.id,
+    submission_id: row.submissionId,
+    site_id: row.siteId,
+    form_type: row.formType,
+    name: row.name,
+    email: row.email,
+    phone: row.phone,
+    message: row.message,
+    extra_fields: row.extraFields,
+    source_url: row.sourceUrl,
+    utm: row.utm,
+    submitted_at: row.submittedAt,
+    status: row.status,
+    notify_status: row.notifyStatus,
+    resend_message_id: row.resendMessageId
+  }).onConflict((oc) => oc.column("id").doNothing()).execute();
+}
+var init_submissions2 = __esm({
+  "src/db/submissions.ts"() {
+    "use strict";
+    init_submission_row();
+  }
+});
+
+// src/db/screenouts.ts
+import { sql as sql2 } from "kysely";
+async function listScreenOutsSince2(db, sinceDate) {
+  const rows = await db.selectFrom("spam_screenouts").select((eb) => [
+    "site_id",
+    eb.fn.sum("honeypot").as("honeypot"),
+    eb.fn.sum("too_fast").as("too_fast"),
+    eb.fn.sum("marked_spam").as("marked_spam")
+  ]).where("date", ">=", sinceDate).groupBy("site_id").execute();
+  const out = /* @__PURE__ */ new Map();
+  for (const r of rows) {
+    out.set(r.site_id, {
+      honeypot: Number(r.honeypot) || 0,
+      tooFast: Number(r.too_fast) || 0,
+      markedSpam: Number(r.marked_spam) || 0
+    });
+  }
+  return out;
+}
+async function backfillScreenoutBucket(db, b) {
+  await sql2`
+    INSERT INTO spam_screenouts (site_id, date, honeypot, too_fast, marked_spam)
+    VALUES (${b.siteId}, ${b.date}, ${b.honeypot}, ${b.tooFast}, ${b.markedSpam})
+    ON CONFLICT (site_id, date) DO UPDATE SET
+      honeypot = excluded.honeypot,
+      too_fast = excluded.too_fast,
+      marked_spam = excluded.marked_spam
+  `.execute(db);
+}
+var init_screenouts2 = __esm({
+  "src/db/screenouts.ts"() {
+    "use strict";
+  }
+});
+
+// src/db/backfill.ts
+var backfill_exports = {};
+__export(backfill_exports, {
+  backfillScreenouts: () => backfillScreenouts,
+  backfillSubmissions: () => backfillSubmissions,
+  reconcile: () => reconcile
+});
+import { sql as sql3 } from "kysely";
+async function backfillSubmissions(base, db) {
+  const rows = [];
+  await base(SUBMISSIONS_TABLE).select({ pageSize: 100 }).eachPage((records, fetchNextPage) => {
+    for (const rec of records) rows.push(mapRow3({ id: rec.id, fields: rec.fields }));
+    fetchNextPage();
+  });
+  for (const row of rows) await backfillSubmission(db, row);
+  return rows.length;
+}
+function num2(v) {
+  return typeof v === "number" && Number.isFinite(v) ? v : 0;
+}
+async function backfillScreenouts(base, db) {
+  const agg = /* @__PURE__ */ new Map();
+  await base(SCREENOUTS_TABLE).select({ pageSize: 100 }).eachPage((records, fetchNextPage) => {
+    for (const rec of records) {
+      const f = rec.fields;
+      const siteId = f["Site"]?.[0] ?? "";
+      const date = typeof f["Date"] === "string" ? f["Date"] : "";
+      if (!siteId || !date) continue;
+      const key = `${siteId} ${date}`;
+      const cur = agg.get(key) ?? { siteId, date, honeypot: 0, tooFast: 0, markedSpam: 0 };
+      cur.honeypot += num2(f["Honeypot"]);
+      cur.tooFast += num2(f["Too-fast"]);
+      cur.markedSpam += num2(f["Marked spam"]);
+      agg.set(key, cur);
+    }
+    fetchNextPage();
+  });
+  for (const bucket of agg.values()) await backfillScreenoutBucket(db, bucket);
+  return agg.size;
+}
+async function reconcile(base, db) {
+  let airtableSubs = 0;
+  await base(SUBMISSIONS_TABLE).select({ pageSize: 100, fields: [] }).eachPage((records, fetchNextPage) => {
+    airtableSubs += records.length;
+    fetchNextPage();
+  });
+  const libCountRow = await sql3`SELECT COUNT(*) AS n FROM submissions`.execute(db);
+  const libsqlSubs = Number(libCountRow.rows[0]?.n ?? 0);
+  const aMap = await listScreenOutsSince(base, "0001-01-01");
+  const lMap = await listScreenOutsSince2(db, "0001-01-01");
+  const sumOf = (m) => {
+    const t = { honeypot: 0, tooFast: 0, markedSpam: 0 };
+    for (const v of m.values()) {
+      t.honeypot += v.honeypot;
+      t.tooFast += v.tooFast;
+      t.markedSpam += v.markedSpam;
+    }
+    return t;
+  };
+  const aScreen = sumOf(aMap);
+  const lScreen = sumOf(lMap);
+  const ok = airtableSubs === libsqlSubs && aScreen.honeypot === lScreen.honeypot && aScreen.tooFast === lScreen.tooFast && aScreen.markedSpam === lScreen.markedSpam;
+  return {
+    ok,
+    submissions: { airtable: airtableSubs, libsql: libsqlSubs },
+    screenouts: { airtable: aScreen, libsql: lScreen }
+  };
+}
+var init_backfill = __esm({
+  "src/db/backfill.ts"() {
+    "use strict";
+    init_submissions();
+    init_screenouts();
+    init_submissions2();
+    init_screenouts2();
+  }
+});
+
 // src/cli/bin.ts
 init_credentials();
 import { dirname as dirname9 } from "path";
@@ -7496,6 +7856,52 @@ async function runGitHubSignalsCommand(opts) {
   };
 }
 
+// src/cli/commands/db.ts
+async function runDbCommand(action, opts) {
+  if (action === "migrate") {
+    const { readDbConfig: readDbConfig2 } = await Promise.resolve().then(() => (init_client2(), client_exports2));
+    const cfg = opts.url ? { url: opts.url } : readDbConfig2();
+    const { runMigrations: runMigrations2 } = await Promise.resolve().then(() => (init_migrate(), migrate_exports));
+    const { createClient: createClient2 } = await import("@libsql/client");
+    const client = createClient2(cfg.url === ":memory:" ? { url: ":memory:" } : cfg);
+    const ran = await runMigrations2(client);
+    return {
+      output: ran.length ? `Applied migrations: ${ran.join(", ")}` : "Already up to date.",
+      code: 0
+    };
+  }
+  if (action === "backfill") {
+    const { openDb: openDb2, readDbConfig: readDbConfig2 } = await Promise.resolve().then(() => (init_client2(), client_exports2));
+    const cfg = opts.url ? { url: opts.url } : readDbConfig2();
+    const { openBase: openBase2, readAirtableConfig: readAirtableConfig2 } = await Promise.resolve().then(() => (init_client(), client_exports));
+    const { backfillSubmissions: backfillSubmissions2, backfillScreenouts: backfillScreenouts2 } = await Promise.resolve().then(() => (init_backfill(), backfill_exports));
+    const base = openBase2(readAirtableConfig2());
+    const db = await openDb2(cfg);
+    const subs = await backfillSubmissions2(base, db);
+    const buckets = await backfillScreenouts2(base, db);
+    return { output: `Backfilled ${subs} submissions, ${buckets} screen-out buckets.`, code: 0 };
+  }
+  if (action === "reconcile") {
+    const { openDb: openDb2, readDbConfig: readDbConfig2 } = await Promise.resolve().then(() => (init_client2(), client_exports2));
+    const cfg = opts.url ? { url: opts.url } : readDbConfig2();
+    const { openBase: openBase2, readAirtableConfig: readAirtableConfig2 } = await Promise.resolve().then(() => (init_client(), client_exports));
+    const { reconcile: reconcile2 } = await Promise.resolve().then(() => (init_backfill(), backfill_exports));
+    const base = openBase2(readAirtableConfig2());
+    const db = await openDb2(cfg);
+    const r = await reconcile2(base, db);
+    const lines = [
+      `submissions: airtable=${r.submissions.airtable} libsql=${r.submissions.libsql}`,
+      `screenouts:  airtable=${JSON.stringify(r.screenouts.airtable)} libsql=${JSON.stringify(r.screenouts.libsql)}`,
+      r.ok ? "OK \u2014 parity confirmed." : "MISMATCH \u2014 do not cut over."
+    ];
+    return { output: lines.join("\n"), code: r.ok ? 0 : 1 };
+  }
+  return {
+    output: `unknown db action '${action}'. Use: migrate | backfill | reconcile.`,
+    code: 1
+  };
+}
+
 // src/cli/version.ts
 import { readFileSync as readFileSync5, existsSync as existsSync4 } from "fs";
 import { dirname as dirname8, join as join28 } from "path";
@@ -7674,6 +8080,12 @@ cli.command(
     opts
   )
 );
+cli.command(
+  "db <action>",
+  "Migrate / backfill / reconcile the libSQL store (migrate | backfill | reconcile)."
+).action(
+  async (action, opts) => runOrExit(() => runDbCommand(action, opts), opts)
+);
 cli.help();
 cli.version(version);
 cli.on("command:*", () => {