@erdoai/cli 0.83.0 → 0.85.0

package/dist/index.js

@@ -1,5 +1,20 @@
 #!/usr/bin/env node
 
+// src/refresh.ts
+function describeRefreshTrigger(state) {
+  if (state.disarmed) return "disarmed \u2014 the experiment has settled";
+  if (state.strategy === "live_webhook") return "incoming webhook events \u2014 page views do not trigger this feed";
+  if (state.strategy === "live_native_poll") return "not running \u2014 native polling is unavailable; configure script_js with a schedule or live_webhook";
+  if (!state.job_id) return "not running \u2014 no linked job; configure the refresh again";
+  if (state.job_enabled === false) return "not running \u2014 the linked job is disabled; configure the refresh again";
+  if (state.schedule) return state.schedule_enabled === false ? `cron ${state.schedule} is disabled` : `cron: ${state.schedule}`;
+  return state.refresh_on_view_enabled !== false ? "runs on view \u2014 when a bound page is opened and the data is stale" : "manual only \u2014 page views do not trigger this refresh";
+}
+function describeRefreshWrites(strategy, mode, key, liveMode) {
+  if (strategy?.startsWith("live_")) return liveMode === "append" ? "append \u2014 each incoming event adds rows" : "replace \u2014 each incoming event replaces the current rows";
+  return mode === "upsert" ? `upsert${key ? ` on ${key}` : ""} \u2014 rows outside what the recipe returns are kept` : "replace \u2014 rows the recipe does not return are deleted";
+}
+
 // src/index.ts
 import { readFileSync as readFileSync4 } from "fs";
 import { basename } from "path";
@@ -1072,6 +1087,35 @@ var ErdoClient = class {
   listDatasetRevisions(slug) {
     return this.request("GET", `/v1/datasets/${encodeURIComponent(slug)}/revisions`);
   }
+  // --- dataset refreshes ---
+  // The configuration that keeps one dataset current — the recipe and the
+  // triggers that run it. A dataset nothing refreshes answers configured:false
+  // rather than 404, so an empty answer is still an answer.
+  getDatasetRefresh(slug) {
+    return this.request(
+      "GET",
+      `/v1/datasets/${encodeURIComponent(slug)}/refresh`
+    );
+  }
+  // Install or replace the whole configuration. Deterministic strategies are
+  // test-run before this returns and the outcome comes back on test_run — a
+  // recipe that saved but cannot run is not a working refresh.
+  setDatasetRefresh(slug, body) {
+    return this.request(
+      "PUT",
+      `/v1/datasets/${encodeURIComponent(slug)}/refresh`,
+      body
+    );
+  }
+  // Run the configured refresh once, now. It is asynchronous: this answers with
+  // the execution it started, and the outcome is read back from
+  // getDatasetRefresh.
+  runDatasetRefresh(slug) {
+    return this.request(
+      "POST",
+      `/v1/datasets/${encodeURIComponent(slug)}/refresh/run`
+    );
+  }
   // --- bounded outreach ---
   putOutreachBatch(batch, body) {
     return this.request(
@@ -1693,6 +1737,12 @@ async function resolveSecretInput(options) {
   return void 0;
 }
 
+// src/integrations.ts
+function formatIntegrationListRow(i) {
+  const provided = i.provided_by_organization_slug ? `	provided by ${i.provided_by_organization_slug}` : "";
+  return `${i.app}	${i.status}	${i.auth_type}	${i.name}	${i.id}${provided}`;
+}
+
 // src/scope.ts
 import { Option } from "commander";
 var ORG_ENV = "ERDO_ORG";
@@ -5681,6 +5731,232 @@ datasetsCmd.command("configure-integration <dataset-id>").description("Set an in
     fail(e);
   }
 });
+var refreshCmd = datasetsCmd.command("refresh").description(
+  "The recipe that keeps a dataset current, and the triggers that run it \u2014 read it with `show`, install or replace it with `set`, and run it once now with `run`"
+);
+var REFRESH_STRATEGIES = ["script", "script_js", "agent_run", "live_webhook"];
+function readRefreshJSONFile(value, label) {
+  return readJSONObject(value.startsWith("@") ? value.slice(1) : value, label);
+}
+refreshCmd.command("show <slug>").description(
+  "Show what keeps a dataset current \u2014 which recipe runs, whether it replaces or merges rows, what triggers it, and how its last run went. A dataset nothing refreshes says so plainly instead of erroring, because 'nothing keeps this current' is the answer most worth reading. The recipe's own source is not printed here; --json carries it."
+).option("--json", "print the raw JSON result instead of a table").action(async (slug, opts) => {
+  try {
+    const res = await new ErdoClient().getDatasetRefresh(slug);
+    if (opts.json) {
+      print(res);
+      return;
+    }
+    if (!res.configured) {
+      console.log(
+        `Nothing keeps ${res.dataset_slug} current \u2014 its rows change only when something writes to it. Install a recipe with: erdo datasets refresh set ${res.dataset_slug} --strategy script_js --transform-js @refresh.js`
+      );
+      return;
+    }
+    const rows = [];
+    rows.push(["strategy", res.strategy || "(none recorded)"]);
+    rows.push(["writes", describeRefreshWrites(res.strategy, res.refresh_mode, res.key_column, res.live_write_mode)]);
+    rows.push(["trigger", describeRefreshTrigger(res)]);
+    if (res.schedule) {
+      const zone = res.schedule_timezone || "UTC";
+      rows.push([
+        "cron",
+        res.schedule_enabled ? `${res.schedule} (${zone})` : `${res.schedule} (${zone}) \u2014 DISARMED, this cron is not firing`
+      ]);
+    } else {
+      rows.push(["cron", "none"]);
+    }
+    rows.push([
+      "on view",
+      res.refresh_on_view_enabled ? `yes \u2014 opening a page bound to this dataset refreshes it once the data is over ${res.refresh_on_view_stale_after_seconds}s old, at most once every ${res.refresh_on_view_debounce_seconds}s` : "no \u2014 opening a page bound to this dataset never triggers a refresh"
+    ]);
+    if (res.disarmed) {
+      rows.push([
+        "disarmed",
+        "yes \u2014 the platform stopped this schedule after the measurement it fed settled"
+      ]);
+    }
+    if (res.last_refresh_at || res.last_refresh_status) {
+      const parts = [res.last_refresh_status || "status not recorded"];
+      if (res.last_refresh_at) parts.push(res.last_refresh_at);
+      if (res.last_refresh_duration_ms !== void 0) {
+        parts.push(`${res.last_refresh_duration_ms} ms`);
+      }
+      rows.push(["last run", parts.join("  ")]);
+      if (res.last_refresh_error) rows.push(["last error", res.last_refresh_error]);
+    } else {
+      rows.push(["last run", "never \u2014 this refresh has not run yet"]);
+    }
+    if (res.live_write_mode) rows.push(["live write mode", res.live_write_mode]);
+    if (res.action_key) rows.push(["action", res.action_key]);
+    if (res.poll_interval_ms !== void 0) {
+      rows.push(["poll interval", `${res.poll_interval_ms} ms`]);
+    }
+    if (res.context_dataset_ids?.length) {
+      rows.push([
+        "context datasets",
+        `${res.context_dataset_ids.length} other dataset(s) the recipe reads`
+      ]);
+    }
+    for (const [label, source] of [
+      ["script (python)", res.script],
+      ["transform_js", res.transform_js],
+      ["agent prompt", res.agent_prompt]
+    ]) {
+      if (source) rows.push(["recipe", `${label}, ${source.length} characters`]);
+    }
+    if (res.job_id) rows.push(["job", res.job_id]);
+    printAlignedTable(["setting", "value"], rows);
+    if (!res.strategy?.startsWith("live_") && res.job_id && res.job_enabled !== false && !res.disarmed && !res.schedule && !res.refresh_on_view_enabled) {
+      console.log(
+        `
+Nothing triggers this refresh on its own \u2014 it runs only when asked: erdo datasets refresh run ${res.dataset_slug}`
+      );
+    }
+    if (res.script || res.transform_js || res.agent_prompt) {
+      console.log(`
+Read the recipe with: erdo datasets refresh show ${res.dataset_slug} --json`);
+    }
+  } catch (e) {
+    fail(e);
+  }
+});
+refreshCmd.command("set <slug>").description(
+  "Install or replace the refresh that keeps a dataset current. This writes the whole configuration, so pass everything the strategy needs in one call \u2014 what you omit is what that strategy does not use. Omitting --schedule is the default and cheapest behaviour: the dataset refreshes when somebody opens a page bound to it and the data is stale, and never runs while nobody is looking; add a cron only when the data must be fresh for something that is not a viewer. --mode replace deletes the rows the recipe did not return, so anything time-series or incremental wants --mode upsert --key-column <c>, which merges by that key and leaves rows outside the returned window alone. Deterministic strategies are test-run before this returns; a recipe that saves but cannot run exits non-zero rather than reporting success."
+).requiredOption(
+  "--strategy <s>",
+  `how the refresh runs: ${REFRESH_STRATEGIES.join(", ")}. script_js is JavaScript and the usual choice for fetch-and-reshape; script is Python, for when the recipe needs pandas or numpy; agent_run hands the job to an LLM agent and is the expensive last resort; live_webhook is a push-driven feed; native polling is unavailable`
+).option(
+  "--mode <replace|upsert>",
+  "replace overwrites every row with what the recipe returned; upsert merges by --key-column and keeps rows outside the returned window. Omit it and you get upsert when you passed --key-column and replace when you did not, since a key column is only meaningful to a merge"
+).option("--key-column <c>", "the column an upsert merges on; required with --mode upsert").option(
+  "--schedule <cron>",
+  "a cron expression to run the refresh on a fixed cadence. Omit it and the refresh runs on view instead \u2014 cheaper, because it never runs while nobody is looking"
+).option("--timezone <tz>", "IANA timezone the cron expression is read in (default UTC)").option(
+  "--script <pyOr@file>",
+  "Python source for the script strategy, inline or @path; it must call emit_dataset(df) or emit_rows(rows)"
+).option("--script-entrypoint <f>", "entrypoint filename for the script strategy (default main.py)").option(
+  "--transform-js <jsOr@file>",
+  "JavaScript source, inline or @path. script_js enters at function refresh(ctx) returning rows; the live strategies enter at function transform(raw) reshaping one event"
+).option("--agent-prompt <textOr@file>", "instructions for the agent_run strategy, inline or @path").option(
+  "--context-datasets <csv>",
+  "slugs of other datasets the recipe reads besides the one it writes"
+).option(
+  "--on-view",
+  "let a page view trigger this refresh (the platform default); pass --no-on-view to turn it off. Sent only when you pass one of the two"
+).option("--no-on-view", "stop page views from triggering this refresh").option(
+  "--stale-after <seconds>",
+  "how old the data must be before a page view triggers a refresh",
+  (v) => parseInt(v, 10)
+).option(
+  "--debounce <seconds>",
+  "minimum gap between two view-triggered refreshes",
+  (v) => parseInt(v, 10)
+).option(
+  "--live-write-mode <append|replace>",
+  "for the live strategies: append for a time-series that accumulates, replace for current state that overwrites on each event"
+).option("--payload-schema <@file>", "JSON file: a schema the transformed row must match").option(
+  "--sample-payload <@file>",
+  "JSON file: an example upstream payload, used to test-run transform_js at configure time"
+).option(
+  "--parameters <@file>",
+  "JSON file: all configuration values for this recipe; omitting clears previous PARAMETERS values"
+).action(
+  async (slug, opts) => {
+    try {
+      if (!REFRESH_STRATEGIES.includes(opts.strategy)) {
+        fail(
+          new Error(
+            `--strategy must be one of: ${REFRESH_STRATEGIES.join(", ")} (got ${JSON.stringify(opts.strategy)})`
+          )
+        );
+        return;
+      }
+      if (opts.mode && opts.mode !== "replace" && opts.mode !== "upsert") {
+        fail(new Error("--mode must be replace or upsert"));
+        return;
+      }
+      if (opts.mode === "upsert" && !opts.keyColumn) {
+        fail(new Error("--mode upsert needs --key-column <c>: the column the merge keys on"));
+        return;
+      }
+      const body = { strategy: opts.strategy };
+      if (opts.mode) body.refresh_mode = opts.mode;
+      if (opts.keyColumn) body.key_column = opts.keyColumn;
+      if (opts.schedule) body.schedule = opts.schedule;
+      if (opts.timezone) body.timezone = opts.timezone;
+      const script = readMaybeFile(opts.script);
+      if (script !== void 0) body.script = script;
+      if (opts.scriptEntrypoint) body.script_entrypoint = opts.scriptEntrypoint;
+      const transformJS = readMaybeFile(opts.transformJs);
+      if (transformJS !== void 0) body.transform_js = transformJS;
+      const agentPrompt = readMaybeFile(opts.agentPrompt);
+      if (agentPrompt !== void 0) body.agent_prompt = agentPrompt;
+      if (opts.contextDatasets) {
+        const slugs = opts.contextDatasets.split(",").map((s) => s.trim()).filter(Boolean);
+        if (slugs.length) body.context_dataset_slugs = slugs;
+      }
+      if (typeof opts.onView === "boolean") body.refresh_on_view_enabled = opts.onView;
+      if (opts.staleAfter !== void 0) body.refresh_on_view_stale_after_seconds = opts.staleAfter;
+      if (opts.debounce !== void 0) body.refresh_on_view_debounce_seconds = opts.debounce;
+      if (opts.liveWriteMode) body.live_write_mode = opts.liveWriteMode;
+      if (opts.payloadSchema) {
+        body.payload_schema = readRefreshJSONFile(opts.payloadSchema, "--payload-schema");
+      }
+      if (opts.samplePayload) {
+        body.sample_payload = readRefreshJSONFile(opts.samplePayload, "--sample-payload");
+      }
+      if (opts.parameters) {
+        body.parameters = readRefreshJSONFile(opts.parameters, "--parameters");
+      }
+      const res = await new ErdoClient().setDatasetRefresh(slug, body);
+      console.log(
+        `Installed a ${res.strategy} refresh on ${res.dataset_slug}${res.previous_strategy ? ` (replacing ${res.previous_strategy})` : ""}`
+      );
+      console.log(`writes: ${describeRefreshWrites(res.strategy, res.refresh_mode, body.key_column, body.live_write_mode)}`);
+      console.log(describeRefreshTrigger({ ...res, refresh_on_view_enabled: body.refresh_on_view_enabled }));
+      if (res.job_id) console.log(`job: ${res.job_id}`);
+      if (res.webhook_url) {
+        console.log(`webhook url: ${res.webhook_url}`);
+        if (res.webhook_secret) {
+          console.log(`webhook secret: ${res.webhook_secret}`);
+          console.log("The secret is shown once, here \u2014 it is never returned again; store it now.");
+        }
+      }
+      if (res.test_run) {
+        if (res.test_run.success) {
+          console.log(`test run: passed in ${res.test_run.duration_ms} ms`);
+        } else {
+          console.error(
+            `test run: FAILED after ${res.test_run.duration_ms} ms \u2014 ${res.test_run.error || "no error reported"}`
+          );
+          console.error(
+            "The configuration is saved but the recipe does not run. Fix it and re-run this command."
+          );
+          process.exitCode = 1;
+        }
+      }
+    } catch (e) {
+      fail(e);
+    }
+  }
+);
+refreshCmd.command("run <slug>").description(
+  "Run a dataset's configured refresh once, now \u2014 the same recipe a cron or a page view would run, without waiting for either. It is asynchronous: this answers with the execution it started, and `datasets refresh show <slug>` carries the outcome once it finishes."
+).option("--json", "print the raw JSON result instead of a summary").action(async (slug, opts) => {
+  try {
+    const res = await new ErdoClient().runDatasetRefresh(slug);
+    if (opts.json) {
+      print(res);
+      return;
+    }
+    console.log(`Started a ${res.strategy} refresh of ${res.dataset_slug} (${res.status})`);
+    console.log(`execution: ${res.job_execution_id}`);
+    console.log(`Check how it went with: erdo datasets refresh show ${res.dataset_slug}`);
+  } catch (e) {
+    fail(e);
+  }
+});
 var analytics = program.command("analytics").description("Page analytics \u2014 what is tracking your published pages, and how they perform with real visitors");
 analytics.command("query <hogql>").description("Run a read-only HogQL query against this org's page-analytics events").option("--json", "print the raw JSON result instead of a table").action(async (hogql, opts) => {
   try {
@@ -5759,13 +6035,10 @@ filterCmd.command("list <slug>").description("List the default filters on a data
   }
 });
 var integrationsCmd = program.command("integrations").description("Connect and inspect integrations");
-integrationsCmd.command("list").description("List connected integrations").action(async () => {
+integrationsCmd.command("list").description("List connected integrations \u2014 app, status, auth type, name, then the connection's id (what --from-connection takes)").action(async () => {
   try {
     const { integrations } = await new ErdoClient().listIntegrations();
-    for (const i of integrations) {
-      const provided = i.provided_by_organization_slug ? `	provided by ${i.provided_by_organization_slug}` : "";
-      console.log(`${i.app}	${i.status}	${i.auth_type}	${i.name}${provided}`);
-    }
+    for (const i of integrations) console.log(formatIntegrationListRow(i));
   } catch (e) {
     fail(e);
   }
@@ -5781,8 +6054,21 @@ integrationsCmd.command("apps [query]").description("Search connectable apps (na
 integrationsCmd.command("connect <app>").description("Connect an app \u2014 pass its API key or other credentials with -c, or omit them to get a browser connect URL for OAuth apps").option("-c, --credential <key=value...>", "credential field, e.g. -c api_key=\u2026 (repeatable); rejected by OAuth apps, which have no credential to pass", (v, acc) => acc.concat(v), []).option("-n, --name <name>", "display name for the connection").option("--rotate", "replace the credentials of the app's existing connection instead of adding a second one \u2014 for a key that was rotated at the provider; requires -c").option("--share", "share the connection with the whole organization instead of keeping it private to you \u2014 for org-level credentials teammates and agents should see").option(
   "--provide-to-managed",
   "let the organizations you manage use this connection as if it were their own \u2014 for a vendor key your agency pays for. Key-authenticated native integrations only, connected in a manager organization"
+).option(
+  "--from-connection <integration-id>",
+  "reuse a credential you verifiably supplied \u2014 find its connection id in column 5 of `erdo integrations list --org <that-org>`. The credential is copied inside Erdo and never shown. Legacy keys without supplier records require an authenticated credential update"
 ).action(async (app, opts) => {
   try {
+    if (opts.fromConnection && opts.credential.length) {
+      fail(new Error("pass either -c or --from-connection, not both \u2014 otherwise the connection would hold a credential you did not choose"));
+    }
+    if (opts.fromConnection && opts.rotate) {
+      fail(
+        new Error(
+          "--from-connection creates a new connection from a credential you already hold, so it cannot be combined with --rotate, which replaces the credential on an existing one"
+        )
+      );
+    }
     let credentials;
     if (opts.credential.length) {
       credentials = {};
@@ -5798,7 +6084,8 @@ integrationsCmd.command("connect <app>").description("Connect an app \u2014 pass
       credentials,
       rotate_credentials: opts.rotate || void 0,
       share_with_org: opts.share || void 0,
-      managed_access: opts.provideToManaged ? "managed_organizations" : void 0
+      managed_access: opts.provideToManaged ? "managed_organizations" : void 0,
+      credentials_from_integration_id: opts.fromConnection || void 0
     });
     print(res);
     const notes = [];

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@erdoai/cli",
-  "version": "0.83.0",
-  "description": "Erdo CLI \u2014 drive datasets, pages, and evals from the terminal or CI",
+  "version": "0.85.0",
+  "description": "Erdo CLI — drive datasets, pages, and evals from the terminal or CI",
   "type": "module",
   "bin": {
     "erdo": "dist/index.js"
@@ -53,4 +53,4 @@
   "overrides": {
     "esbuild": "^0.28.1"
   }
-}
+}