@oalo-ai/cli 0.0.0-stage → 0.1.0

package/src/commands/data.js

@@ -0,0 +1,446 @@
+import { readFileSync } from "node:fs";
+import { basename } from "node:path";
+import { OaloChannel } from "../channel.js";
+import { resolve as resolveConfig, controlPlane, HOW_TO_CONNECT } from "../config.js";
+import { requireFeature } from "../server.js";
+import { ApiError } from "../api.js";
+import { dim } from "../events.js";
+import {
+  parseRecords,
+  toRows,
+  targetEnv,
+  listModels,
+  routes,
+  listRows,
+  getRow,
+  watchRows,
+  pushRows,
+  deleteRow,
+  writeTokenFor,
+  parseFields,
+  createModel,
+  promoteModel,
+} from "../data.js";
+
+/**
+ * `oalo data`: models and rows in this workspace.
+ *
+ *   oalo data models                         list models (fields, owner field)
+ *   oalo data models create <name> --field total:float [--field ...] [--owned-by user_id]
+ *   oalo data list <table> [--since N] [--limit N]
+ *   oalo data get <table> <id>
+ *   oalo data push <table> <file> [--id-field id] [--format csv|json|jsonl] [--types] [--from N] [--dry-run]
+ *   oalo data delete <table> <id>
+ *   oalo data watch <table> [--since N]
+ *   oalo data promote <table>                copy dev rows to prod
+ *   oalo data routes                         the Sync API's endpoints, scopes and limits
+ *   oalo data events <table> created [updated] | --off
+ *                                            workflow events when a live row changes
+ *   oalo data vectorize <table> [--fields a,b] [--where '...'] [--describe '...'] [--id col] [--off]
+ *                                            make a table searchable by bots
+ *   oalo data keep-from-ai <table> [a,b] [--off]
+ *                                            fields a bot's data tools never hand its AI model
+ *
+ * Every command works on one schema, `dev` or `prod` (`--env`, default the
+ * project's link, default dev), and says which. Rows written to prod sync to
+ * live apps at once, so writing there needs `--env prod` on the command line.
+ */
+export async function data(flags, rest, opts = {}) {
+  const sub = rest[1];
+  const cfg = resolveConfig(flags);
+
+  try {
+    switch (sub) {
+      case "models":
+        return rest[2] === "create" ? await modelsCreate(flags, rest, cfg) : await models(flags, cfg);
+      case "list":
+        return await list(flags, rest, cfg);
+      case "get":
+        return await get(flags, rest, cfg);
+      case "push":
+        return await push(flags, rest, cfg);
+      case "delete":
+        return await del(flags, rest, cfg);
+      case "watch":
+        return await watch(flags, rest, cfg);
+      case "promote":
+        return await promote(flags, rest, cfg);
+      case "routes":
+        return await showRoutes(flags, cfg);
+      case "events":
+        return await rowEvents(flags, rest, cfg);
+      case "vectorize":
+        return await vectorize(flags, rest, cfg, opts);
+      case "keep-from-ai":
+        return await keepFromAi(flags, rest, cfg, opts);
+      default:
+        usage(sub ? `unknown data command: ${sub}` : null);
+    }
+  } catch (e) {
+    console.error(explain(e));
+    process.exitCode = e.exitCode || process.exitCode || 1;
+  }
+}
+
+function usage(problem) {
+  if (problem?.startsWith("usage:")) {
+    console.error(problem);
+    process.exitCode = 2;
+    return;
+  }
+  if (problem) console.error(problem + "\n");
+  console.error(
+    [
+      "usage: oalo data models | models create <name> --field name:type ...",
+      "       oalo data list <table> | get <table> <id> | watch <table>",
+      "       oalo data push <table> <file> [--id-field id] [--dry-run]",
+      "       oalo data delete <table> <id> | promote <table> | routes",
+      "       oalo data events <table> created [updated] | --off",
+      "       oalo data vectorize <table> [--fields a,b] [--where '...'] [--describe '...'] [--off]",
+      "       oalo data keep-from-ai <table> [a,b] [--off]",
+      "",
+      "Add --env dev|prod to pick the schema (default: the project's link, else dev). `oalo help data` has more.",
+    ].join("\n"),
+  );
+  process.exitCode = 2;
+}
+
+// A 403 on a read almost always means the token can't see that table.
+function explain(e) {
+  if (e instanceof ApiError && e.status === 403) return `${e.message}. The saved data token can't do this; check \`oalo whoami\`, or ask an admin for access to this table.`;
+  if (e instanceof ApiError && e.status === 404) return `${e.message}. Check the table name with \`oalo data models\`.`;
+  return e.message;
+}
+
+/**
+ * `oalo data events <table> created updated`: a live table's changes start
+ * workflows (data.<table>.created / .updated), whoever writes it: the app's
+ * own SQL, the Sync API, the dashboard. Live rows only. `--off` stops them.
+ */
+async function rowEvents(flags, rest, cfg) {
+  const table = rest[2];
+  const kinds = rest.slice(3);
+  if (!table || (kinds.length === 0 && !flags.off) || (kinds.length && flags.off)) {
+    return usage("usage: oalo data events <table> created [updated]   or   oalo data events <table> --off");
+  }
+  const bad = kinds.filter((k) => !["created", "updated"].includes(k));
+  if (bad.length) return usage(`usage: the kinds are created and updated (got ${bad.join(", ")})`);
+
+  const { ch, url } = await control(cfg, "oalo data events");
+  try {
+    await requireFeature(url, "row_events", "oalo data events");
+    const r = await ch.call("data:set_row_events", { table, kinds: flags.off ? [] : kinds });
+    if (flags.json) return void console.log(JSON.stringify(r, null, 2));
+    if (!r.row_events.length) return void console.log(`${r.table}: no workflow events`);
+    console.log(`${r.table}: ${r.row_events.map((k) => `data.${r.table}.${k}`).join(", ")}`);
+    console.log(dim("Fired when a live row changes, by anything that writes it. Pick them as a workflow's trigger."));
+  } finally {
+    ch.close();
+  }
+}
+
+const VECTORIZE_USAGE = "usage: oalo data vectorize <table> [--fields a,b] [--where 'status in active,draft'] [--describe '...'] [--id <column>]   or   --off";
+
+/**
+ * `oalo data vectorize <table> --fields title,body`: bots can search the
+ * table once it's attached to them (`oalo bots knowledge`). An Oalo model or a
+ * table the app made; the server decides which. With no flags, its status.
+ */
+async function vectorize(flags, rest, cfg, opts) {
+  const table = rest[2];
+  if (!table || rest.length > 3) return usage(VECTORIZE_USAGE);
+  const list = (v) => String(v).split(",").map((s) => s.trim()).filter(Boolean);
+  for (const f of ["fields", "where", "describe", "id"]) {
+    if (flags[f] === true) return usage(`usage: --${f} needs a value. ${VECTORIZE_USAGE}`);
+  }
+  if (flags.off && (flags.fields || flags.where)) return usage(`usage: --off can't go with --fields or --where. ${VECTORIZE_USAGE}`);
+
+  const env = flags.env || cfg.env || "dev";
+  if (!["dev", "prod"].includes(env)) return usage(`usage: --env must be dev or prod (got ${env})`);
+  const params = { table, env };
+  if (flags.fields) params.fields = list(flags.fields);
+  if (flags.where) params.where = flags.where;
+  if (flags.describe) params.description = flags.describe;
+  if (flags.id) params.id_col = flags.id;
+  if (flags.off) params.off = true;
+  const changing = Object.keys(params).length > 2;
+
+  const { ch, url } = await control(cfg, "oalo data vectorize", { channel: opts.channel });
+  let r;
+  try {
+    if (!opts.channel) await requireFeature(url, "knowledge", "oalo data vectorize");
+    r = await ch.call("imports:searchable", params);
+  } finally {
+    ch.close();
+  }
+  if (flags.json) return void console.log(JSON.stringify(r, null, 2));
+  printSearchable(r, changing);
+}
+
+function printSearchable(r, changed) {
+  const name = `${r.table} (${r.kind === "model" ? "model" : "table"})`;
+  if (!r.enabled) {
+    console.log(changed ? `${name}: no longer searchable. Bots it's attached to stop finding it.` : `${name}: not searchable by bots.`);
+    if (!changed) console.log(dim(`Turn it on: oalo data vectorize ${r.table} --fields <field,...>`));
+    return;
+  }
+  console.log(`${name}: bots can search it once attached, as ${r.handle}`);
+  console.log(`  embeds: ${r.fields.join(", ")}`);
+  if (r.where) console.log(`  only rows where: ${r.where}`);
+  if (r.description) console.log(`  described as: ${r.description}`);
+  const rows = Object.entries(r.rows || {}).map(([env, c]) => `${env} ${c.embedded} of ${c.total}`);
+  if (rows.length) console.log(`  rows embedded: ${rows.join(", ")}`);
+  if (r.enqueued) console.log(dim(`Embedding ${r.enqueued} row${r.enqueued === 1 ? "" : "s"} now. New and edited rows follow within 15 minutes.`));
+  if (r.kept_from_ai?.length) console.log(`  kept from AI: ${r.kept_from_ai.join(", ")}`);
+  console.log(
+    dim(
+      `The embedding provider (Voyage) gets the text of ${r.fields.join(", ")}. A bot's answer sends the rows its search finds to its AI model` +
+        (r.kept_from_ai?.length ? `, without ${r.kept_from_ai.join(", ")}.` : `. Keep fields out of that: oalo data keep-from-ai ${r.table} <field,...>`),
+    ),
+  );
+  console.log(dim(`Attach it to a bot: oalo bots knowledge <bot> --add ${r.handle}`));
+}
+
+const KEEP_USAGE = "usage: oalo data keep-from-ai <table> [field,...] [--off]";
+
+/**
+ * `oalo data keep-from-ai <table> cost,notes`: those fields stay stored and
+ * synced, and workflows read them, but a bot's data tools never hand them to
+ * its AI model, and they can't be embedded. `--off` with fields takes those
+ * off the list; `--off` alone clears it. With neither, the list.
+ */
+async function keepFromAi(flags, rest, cfg, opts) {
+  const table = rest[2];
+  if (!table || rest.length > 4) return usage(KEEP_USAGE);
+  const fields = rest[3] ? rest[3].split(",").map((s) => s.trim()).filter(Boolean) : [];
+  const env = flags.env || cfg.env || "dev";
+  if (!["dev", "prod"].includes(env)) return usage(`usage: --env must be dev or prod (got ${env})`);
+
+  const { ch } = await control(cfg, "oalo data keep-from-ai", { channel: opts.channel });
+  let r;
+  try {
+    r =
+      fields.length || flags.off
+        ? await ch.call("data:keep_from_ai", { table, env, fields, off: flags.off === true })
+        : await ch.call("imports:searchable", { table, env });
+  } finally {
+    ch.close();
+  }
+  if (flags.json) return void console.log(JSON.stringify(r, null, 2));
+  const kept = r.kept_from_ai || [];
+  const name = `${r.table} (${r.kind === "model" ? "model" : `${env} table`})`;
+  if (!kept.length) return void console.log(`${name}: every field can reach a bot's AI model.${fields.length || flags.off ? "" : dim(` Keep some out: oalo data keep-from-ai ${r.table} <field,...>`)}`);
+  console.log(`${name}: kept from AI: ${kept.join(", ")}`);
+  console.log(dim("Stored and synced as before, and workflows read them. A bot's data tools leave them out, and they can't be embedded."));
+}
+
+/** The read side: the saved sync token, on the chosen schema. */
+function readCfg(flags, cfg) {
+  if (!cfg.serverUrl || !cfg.token) {
+    // Exit 2, as the help says for "no token" (audit C2).
+    throw Object.assign(new Error(`no workspace or data token for this project.\n\n${HOW_TO_CONNECT}`), { exitCode: 2 });
+  }
+  const env = flags.env || cfg.env || "dev";
+  if (!["dev", "prod"].includes(env)) throw new Error(`--env must be dev or prod (got ${env})`);
+  return { serverUrl: cfg.serverUrl, token: cfg.token, env };
+}
+
+/**
+ * The control channel, for writes made as the person. `needsFeature` is for the
+ * model commands, whose events are newer than the token events row writes use.
+ */
+async function control(cfg, what, { needsFeature = false, channel } = {}) {
+  const { url, token } = controlPlane(cfg);
+  if (!url || !token) throw new Error(`${what} needs a CLI token.\n\n${HOW_TO_CONNECT}`);
+  if (needsFeature) await requireFeature(url, "data", what);
+  const ch = (channel || ((o) => new OaloChannel(o)))({ url, token });
+  await ch.connect();
+  return { ch, url };
+}
+
+const out = (flags, value, human) => (flags.json ? console.log(JSON.stringify(value, null, 2)) : human());
+const where = (c) => dim(`${c.serverUrl} (${c.env})`);
+
+// ── models ───────────────────────────────────────────────────────────────
+
+async function models(flags, cfg) {
+  const c = readCfg(flags, cfg);
+  const m = await listModels(c);
+  out(flags, m, () => {
+    const names = Object.keys(m.models || {}).sort();
+    console.log(`${names.length} model${names.length === 1 ? "" : "s"} in ${where(c)}`);
+    for (const name of names) {
+      const model = m.models[name];
+      const fields = Array.isArray(model.fields)
+        ? model.fields.map((f) => `${f.name}:${f.type}`)
+        : Object.entries(model.fields || {}).map(([k, v]) => `${k}:${typeof v === "string" ? v : v?.type || "?"}`);
+      const who = model.owner_field ? `owned by ${model.owner_field}` : model.shared ? "shared" : "";
+      console.log(`  ${name}${who ? dim(`  ${who}`) : ""}`);
+      if (fields.length) console.log(dim(`    ${fields.join("  ")}`));
+    }
+  });
+}
+
+async function modelsCreate(flags, rest, cfg) {
+  const name = rest[3];
+  const fields = parseFields(flags.field || []);
+  if (!name || !fields.length) {
+    return usage("usage: oalo data models create <name> --field name:type [--field ...] [--owned-by user_id]\n" +
+      "types: string text integer float boolean datetime date json list references:<table>");
+  }
+  const { ch, url } = await control(cfg, "oalo data models create", { needsFeature: true });
+  try {
+    const r = await createModel(ch, { name, fields, ownedBy: typeof flags["owned-by"] === "string" ? flags["owned-by"] : null });
+    out(flags, r, () => {
+      console.log(`created ${name} in ${url} (dev and prod): ${fields.map((f) => `${f.name}:${f.type}`).join(", ")}`);
+      console.log(dim("Rows go in with `oalo data push`; regenerate bindings with `oalo typegen`."));
+    });
+  } finally {
+    ch.close();
+  }
+}
+
+async function promote(flags, rest, cfg) {
+  const table = rest[2];
+  if (!table) return usage("usage: oalo data promote <table>");
+  const { ch, url } = await control(cfg, "oalo data promote", { needsFeature: true });
+  try {
+    const r = await promoteModel(ch, table);
+    out(flags, r, () => console.log(`copying ${table} rows from dev to prod in ${url}. Live apps get them as they land.`));
+  } finally {
+    ch.close();
+  }
+}
+
+// ── rows ─────────────────────────────────────────────────────────────────
+
+async function list(flags, rest, cfg) {
+  const table = rest[2];
+  if (!table) return usage("usage: oalo data list <table> [--since N] [--limit N]");
+  const c = readCfg(flags, cfg);
+  const page = await listRows(c, table, {
+    since: flags.since,
+    limit: flags.limit,
+    includeDeleted: !!flags["include-deleted"],
+    order: flags.order,
+  });
+  out(flags, page, () => {
+    for (const row of page.rows || []) console.log(JSON.stringify({ id: row.id, ...row.document }));
+    console.error(dim(`${(page.rows || []).length} rows from ${table} in ${c.serverUrl} (${c.env})` +
+      (page.has_more ? `; more with --since ${page.next_since}` : "")));
+  });
+}
+
+async function get(flags, rest, cfg) {
+  const [table, id] = [rest[2], rest[3]];
+  if (!table || !id) return usage("usage: oalo data get <table> <id>");
+  const row = await getRow(readCfg(flags, cfg), table, id);
+  console.log(JSON.stringify(flags.json ? row : { id: row.id, ...row.document }, null, 2));
+}
+
+async function watch(flags, rest, cfg) {
+  const table = rest[2];
+  if (!table) return usage("usage: oalo data watch <table> [--since N]");
+  const c = readCfg(flags, cfg);
+  console.error(dim(`watching ${table} in ${c.serverUrl} (${c.env}). Ctrl-C to stop.`));
+  const ac = new AbortController();
+  process.on("SIGINT", () => ac.abort());
+  try {
+    await watchRows(c, table, {
+      since: flags.since || 0,
+      signal: ac.signal,
+      onEvent: (event, d) => {
+        if (event === "heartbeat") return;
+        console.log(flags.json ? JSON.stringify({ event, data: d }) : `${event} ${JSON.stringify(d?.id ? { id: d.id, ...d.document } : d)}`);
+      },
+    });
+  } catch (e) {
+    if (e.name !== "AbortError") throw e;
+  }
+}
+
+async function push(flags, rest, cfg) {
+  const [table, file] = [rest[2], rest[3]];
+  if (!table || !file) return usage("usage: oalo data push <table> <file.csv|.json|.jsonl> [--id-field id] [--dry-run]");
+
+  const env = targetEnv(flags, cfg);
+  const idField = typeof flags["id-field"] === "string" ? flags["id-field"] : "id";
+  const records = parseRecords(readFileSync(file, "utf8"), {
+    format: typeof flags.format === "string" ? flags.format : undefined,
+    name: basename(file),
+    types: !!flags.types,
+  });
+  const { rows, missing } = toRows(records, { idField, newIds: !!flags["new-ids"] });
+  const from = Number(flags.from || 0);
+  const todo = rows.slice(from);
+
+  if (missing.length) {
+    console.error(`${missing.length} record${missing.length === 1 ? "" : "s"} have no ${idField} and will be skipped (first: #${missing[0].index + 1}). ` +
+      `Pick the id column with --id-field, or let the server make ids with --new-ids (a re-send then adds copies).`);
+  }
+
+  // Check the table exists before writing a single row.
+  const read = readCfg({ ...flags, env }, cfg);
+  const manifest = await listModels(read);
+  if (!manifest.models?.[table]) {
+    throw new Error(`there is no model named ${table} in ${cfg.serverUrl}. Create it first: oalo data models create ${table} --field name:type ...`);
+  }
+
+  if (flags["dry-run"]) {
+    out(flags, { table, env, rows: todo.length, skipped: missing.length, first: todo[0] ?? null }, () => {
+      console.log(`${todo.length} rows would be upserted into ${table} in ${cfg.serverUrl} (${env}), by ${idField}.`);
+      if (todo[0]) console.log(dim(`first: ${JSON.stringify(todo[0])}`));
+      console.log(dim("dry run: nothing was written"));
+    });
+    return;
+  }
+
+  const { ch, url } = await control(cfg, "oalo data push");
+  let token;
+  try {
+    token = await writeTokenFor(url, [table], ch, { log: (m) => console.error(dim(m)) });
+  } finally {
+    ch.close();
+  }
+
+  console.error(dim(`upserting ${todo.length} rows into ${table} in ${url} (${env}), by ${idField}`));
+  let result;
+  try {
+    result = await pushRows({ serverUrl: url, token, env }, table, todo, {
+      onBatch: ({ done, total }) => process.stderr.write(dim(`  ${from + done}/${from + total}\n`)),
+    });
+  } catch (e) {
+    throw new Error(`${e.message}\nRows up to the last progress line were written. Writes are upserts by ${idField}, so it is safe to re-run, or resume with --from <that number>.`);
+  }
+
+  out(flags, { table, env, ...result, skipped: missing.length }, () => {
+    console.log(`${result.written} rows written to ${table} (${env})${result.failed.length ? `, ${result.failed.length} rejected` : ""}.`);
+    for (const f of result.failed.slice(0, 5)) console.log(`  ${f.id}: ${f.error}`);
+    if (result.failed.length > 5) console.log(dim(`  ...and ${result.failed.length - 5} more (--json lists them all)`));
+    if (env === "dev") console.log(dim(`Live apps read prod. When dev looks right: oalo data promote ${table}`));
+  });
+  if (result.failed.length) process.exitCode = 1;
+}
+
+async function del(flags, rest, cfg) {
+  const [table, id] = [rest[2], rest[3]];
+  if (!table || !id) return usage("usage: oalo data delete <table> <id>");
+  const env = targetEnv(flags, cfg);
+  const { ch, url } = await control(cfg, "oalo data delete");
+  let token;
+  try {
+    token = await writeTokenFor(url, [table], ch, { log: (m) => console.error(dim(m)) });
+  } finally {
+    ch.close();
+  }
+  const r = await deleteRow({ serverUrl: url, token, env }, table, id);
+  out(flags, r, () => console.log(`deleted ${table}/${id} in ${url} (${env}). It stays in the table, marked deleted, and leaves synced apps.`));
+}
+
+async function showRoutes(flags, cfg) {
+  const r = await routes(readCfg(flags, cfg));
+  out(flags, r, () => {
+    for (const e of r.endpoints || []) console.log(`${e.method.padEnd(6)} ${e.path.padEnd(32)} ${dim(`[${e.scope}] ${e.description}`)}`);
+    if (r.rate_limits_per_minute) console.log(dim(`\nlimits per minute: ${Object.entries(r.rate_limits_per_minute).map(([k, v]) => `${k} ${v}`).join(", ")}`));
+  });
+}

package/src/commands/db.js

@@ -0,0 +1,58 @@
+import { createWriteStream, existsSync, rmSync, statSync } from "node:fs";
+import { resolve as resolveConfig, controlPlane, HOW_TO_CONNECT } from "../config.js";
+import { dim } from "../events.js";
+
+/**
+ * `oalo db dump [--env prod] [--out file.sql]`: all of an env's data as plain
+ * SQL (every app's own tables and the Oalo models, with their rows), so it can
+ * be restored anywhere with `psql -f`. Workspace admins only. The dump ends
+ * with pg_dump's own "dump complete" line; a dump without it is cut off, and
+ * the file is removed rather than left looking whole.
+ */
+export async function db(flags, rest, opts = {}) {
+  const sub = rest[1];
+  if (sub !== "dump") return fail("usage: oalo db dump [--env dev|prod] [--out <file.sql>]", 2);
+  const env = flags.env === "dev" ? "dev" : "prod";
+  const { url, token } = controlPlane(resolveConfig(flags));
+  if (!url || !token) return fail(`oalo db dump needs a CLI token.\n\n${HOW_TO_CONNECT}`, 2);
+
+  const host = new URL(url).hostname.split(".")[0];
+  const out = typeof flags.out === "string" ? flags.out : `${host}-${env}-${new Date().toISOString().slice(0, 10)}.sql`;
+  if (existsSync(out) && !flags.force) return fail(`${out} exists. Choose another --out, or add --force to replace it.`, 2);
+
+  const doFetch = opts.fetch || fetch;
+  let res;
+  try {
+    res = await doFetch(`${url.replace(/\/$/, "")}/api/cli/v1/db/dump?env=${env}`, { headers: { authorization: `Bearer ${token}` } });
+  } catch (e) {
+    return fail(`could not reach ${url} (${e.message})`, 1);
+  }
+  if (!res.ok) {
+    const body = await res.json().catch(() => ({}));
+    return fail(body?.error?.message || `the dump was refused (HTTP ${res.status})`, 1);
+  }
+
+  const file = createWriteStream(out);
+  let tail = "";
+  const reader = res.body.getReader();
+  const decoder = new TextDecoder();
+  for (;;) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    file.write(value);
+    tail = (tail + decoder.decode(value, { stream: true })).slice(-400);
+  }
+  await new Promise((r) => file.end(r));
+
+  if (!/PostgreSQL database dump complete/.test(tail)) {
+    rmSync(out, { force: true });
+    return fail(`the dump was cut off before the end, so ${out} was removed. Run it again.`, 1);
+  }
+  const mb = (statSync(out).size / 1048576).toFixed(1);
+  console.log(`${out}  ${dim(`${mb} MB, the ${env} data as SQL. Restore it with: psql "$DATABASE_URL" -f ${out}`)}`);
+}
+
+function fail(message, code) {
+  console.error(message);
+  process.exitCode = code;
+}