@withone/cli 1.37.2 → 1.38.0

package/README.md

@@ -343,7 +343,7 @@ one sync run stripe --full-refresh
 | `schedule add/list/status/remove/repair` | Cron-backed scheduled syncs with drift detection |
 | `remove <platform>` | Delete local data (`--dry-run` to preview) |
 
-Change hooks (`onInsert`, `onUpdate`, `onChange`) fire per-page during sync — pipe to a shell command, a flow, or an event log. Run `one guide sync` for the full reference.
+Change hooks (`onInsert`, `onUpdate`, `onChange`) fire per-page during sync — pipe to a shell command, a flow, or an event log. Root-array responses (e.g. Hacker News `/v0/topstories.json` → `[9129911, 9129199, ...]`) are supported by setting `resultsPath` to `""`, `"$"`, or `"."`; primitive elements are auto-wrapped as `{ [idField]: value }`. Run `one guide sync` for the full reference.
 
 ### `one guide [topic]`
 

package/dist/index.js

@@ -4727,12 +4727,15 @@ function readProfile(platform, model) {
   }
 }
 function writeProfile(profile) {
-  const required = ["platform", "model", "connectionKey", "actionId", "resultsPath", "idField", "pagination"];
+  const required = ["platform", "model", "connectionKey", "actionId", "idField", "pagination"];
   for (const field of required) {
     if (!profile[field]) {
       throw new Error(`Missing required field: ${field}`);
     }
   }
+  if (profile.resultsPath === void 0) {
+    throw new Error('Missing required field: resultsPath (use "" or "$" for root-array responses)');
+  }
   if (!profile.pagination.type) {
     throw new Error("Missing required field: pagination.type");
   }
@@ -4853,30 +4856,82 @@ function handleId(response, config2, records) {
 import fs9 from "fs";
 import path9 from "path";
 var SYNC_DIR = path9.join(".one", "sync");
-var STATE_FILE = path9.join(SYNC_DIR, "sync_state.json");
-function readSyncState() {
+var STATE_DIR = path9.join(SYNC_DIR, "state");
+var LEGACY_STATE_FILE = path9.join(SYNC_DIR, "sync_state.json");
+function modelFilePath(platform, model) {
+  return path9.join(STATE_DIR, platform, `${model}.json`);
+}
+function migrateLegacyIfNeeded() {
+  if (!fs9.existsSync(LEGACY_STATE_FILE)) return;
   try {
-    if (!fs9.existsSync(STATE_FILE)) return {};
-    const raw = fs9.readFileSync(STATE_FILE, "utf-8");
-    return JSON.parse(raw);
+    const raw = fs9.readFileSync(LEGACY_STATE_FILE, "utf-8");
+    const legacy = JSON.parse(raw);
+    for (const [platform, models] of Object.entries(legacy)) {
+      for (const [model, modelState] of Object.entries(models)) {
+        if (fs9.existsSync(modelFilePath(platform, model))) continue;
+        writeModelFile(platform, model, modelState);
+      }
+    }
+    fs9.unlinkSync(LEGACY_STATE_FILE);
   } catch {
-    return {};
+    try {
+      fs9.unlinkSync(LEGACY_STATE_FILE);
+    } catch {
+    }
   }
 }
-function writeSyncState(state) {
-  fs9.mkdirSync(SYNC_DIR, { recursive: true });
-  const tmp = STATE_FILE + ".tmp";
+function writeModelFile(platform, model, state) {
+  const dir = path9.join(STATE_DIR, platform);
+  fs9.mkdirSync(dir, { recursive: true });
+  const file = path9.join(dir, `${model}.json`);
+  const tmp = `${file}.${process.pid}.${Math.random().toString(36).slice(2, 10)}.tmp`;
   fs9.writeFileSync(tmp, JSON.stringify(state, null, 2));
-  fs9.renameSync(tmp, STATE_FILE);
+  fs9.renameSync(tmp, file);
+}
+function readModelFile(platform, model) {
+  try {
+    const raw = fs9.readFileSync(modelFilePath(platform, model), "utf-8");
+    return JSON.parse(raw);
+  } catch {
+    return null;
+  }
+}
+function readSyncState() {
+  migrateLegacyIfNeeded();
+  const result = {};
+  let platforms;
+  try {
+    platforms = fs9.readdirSync(STATE_DIR);
+  } catch {
+    return result;
+  }
+  for (const platform of platforms) {
+    const platformDir = path9.join(STATE_DIR, platform);
+    let entries;
+    try {
+      entries = fs9.readdirSync(platformDir);
+    } catch {
+      continue;
+    }
+    for (const entry of entries) {
+      if (!entry.endsWith(".json")) continue;
+      const model = entry.slice(0, -".json".length);
+      const modelState = readModelFile(platform, model);
+      if (modelState) {
+        if (!result[platform]) result[platform] = {};
+        result[platform][model] = modelState;
+      }
+    }
+  }
+  return result;
 }
 function getModelState(platform, model) {
-  const state = readSyncState();
-  return state[platform]?.[model] ?? null;
+  migrateLegacyIfNeeded();
+  return readModelFile(platform, model);
 }
 function updateModelState(platform, model, partial) {
-  const state = readSyncState();
-  if (!state[platform]) state[platform] = {};
-  const existing = state[platform][model] ?? {
+  migrateLegacyIfNeeded();
+  const existing = readModelFile(platform, model) ?? {
     lastSync: null,
     lastCursor: null,
     totalRecords: 0,
@@ -4884,21 +4939,26 @@ function updateModelState(platform, model, partial) {
     since: null,
     status: "idle"
   };
-  state[platform][model] = { ...existing, ...partial };
-  writeSyncState(state);
+  writeModelFile(platform, model, { ...existing, ...partial });
 }
 function removeModelState(platform, model) {
-  const state = readSyncState();
-  if (!state[platform]) return;
+  migrateLegacyIfNeeded();
+  const platformDir = path9.join(STATE_DIR, platform);
   if (model) {
-    delete state[platform][model];
-    if (Object.keys(state[platform]).length === 0) {
-      delete state[platform];
+    try {
+      fs9.unlinkSync(modelFilePath(platform, model));
+    } catch {
+    }
+    try {
+      if (fs9.readdirSync(platformDir).length === 0) fs9.rmdirSync(platformDir);
+    } catch {
     }
   } else {
-    delete state[platform];
+    try {
+      fs9.rmSync(platformDir, { recursive: true, force: true });
+    } catch {
+    }
   }
-  writeSyncState(state);
 }
 
 // src/lib/sync/db.ts
@@ -5637,6 +5697,50 @@ function detectColumnType2(value) {
   return "TEXT";
 }
 
+// src/lib/sync/extract.ts
+var ROOT_PATH_TOKENS = /* @__PURE__ */ new Set(["", "$", "."]);
+function isRootPath(resultsPath) {
+  return resultsPath === void 0 || resultsPath === null || ROOT_PATH_TOKENS.has(resultsPath);
+}
+function isPrimitive(value) {
+  const t = typeof value;
+  return t === "string" || t === "number" || t === "boolean";
+}
+function describeType(value) {
+  if (value === null) return "null";
+  if (Array.isArray(value)) return "array";
+  return typeof value;
+}
+function extractRecords(responseData, resultsPath, idField, profileLabel) {
+  let candidate;
+  if (isRootPath(resultsPath)) {
+    candidate = responseData;
+  } else {
+    candidate = getByDotPath(responseData, resultsPath);
+  }
+  if (!Array.isArray(candidate)) {
+    const topKeys = typeof responseData === "object" && responseData !== null && !Array.isArray(responseData) ? Object.keys(responseData).slice(0, 10) : [];
+    const pathLabel = isRootPath(resultsPath) ? "<root>" : `'${resultsPath}'`;
+    const typeLabel = describeType(responseData);
+    const keyHint = topKeys.length > 0 ? ` Top-level keys: [${topKeys.join(", ")}].` : "";
+    throw new Error(
+      `Could not find results at path ${pathLabel} for profile ${profileLabel}. Response top-level type is ${typeLabel}.${keyHint} Set resultsPath to the array field, or use "" / "$" / "." for root arrays.`
+    );
+  }
+  if (candidate.length === 0) {
+    return { records: [], wrappedPrimitives: false };
+  }
+  if (isPrimitive(candidate[0])) {
+    const wrapped = [];
+    for (const value of candidate) {
+      if (!isPrimitive(value)) continue;
+      wrapped.push({ [idField]: String(value) });
+    }
+    return { records: wrapped, wrappedPrimitives: true };
+  }
+  return { records: candidate, wrappedPrimitives: false };
+}
+
 // src/lib/sync/runner.ts
 var MAX_RETRIES_PER_PAGE = 3;
 var DEFAULT_SINCE_DAYS = 90;
@@ -5881,13 +5985,13 @@ async function syncModel(api, profile, options) {
           throw err;
         }
       }
-      const records = getByDotPath(responseData, profile.resultsPath);
-      if (!Array.isArray(records)) {
-        const topKeys = typeof responseData === "object" && responseData !== null ? Object.keys(responseData) : [];
-        throw new Error(
-          `Could not find results at path '${profile.resultsPath}' in API response. Check your sync profile. Response keys: [${topKeys.join(", ")}]`
-        );
-      }
+      const extraction = extractRecords(
+        responseData,
+        profile.resultsPath,
+        profile.idField,
+        `${platform}/${model}`
+      );
+      const records = extraction.records;
       if (records.length === 0 && page === 0) {
         if (!isAgentMode()) {
           process.stderr.write(`No records found for ${model} with the given filters.
@@ -6186,23 +6290,36 @@ async function testSyncProfile(api, profile) {
     return report;
   }
   let resolvedResultsPath = profile.resultsPath;
-  let records = getByDotPath(responseData, resolvedResultsPath);
-  if (!Array.isArray(records) && typeof responseData === "object" && responseData !== null) {
-    const topObj = responseData;
-    const arrayKey = Object.keys(topObj).find((k) => Array.isArray(topObj[k]));
-    if (arrayKey) {
-      resolvedResultsPath = arrayKey;
-      records = topObj[arrayKey];
+  const rawCandidate = isRootPath(resolvedResultsPath) ? responseData : getByDotPath(responseData, resolvedResultsPath);
+  let records = Array.isArray(rawCandidate) ? rawCandidate : null;
+  if (records === null) {
+    if (Array.isArray(responseData)) {
+      resolvedResultsPath = "";
+      records = responseData;
       report.autoFixed = report.autoFixed ?? {};
-      report.autoFixed.resultsPath = arrayKey;
+      report.autoFixed.resultsPath = "";
       checks.push({
-        name: `resultsPath auto-discovered \u2192 "${arrayKey}"`,
+        name: `resultsPath auto-discovered \u2192 <root>`,
         ok: true,
-        detail: `Profile had "${profile.resultsPath}" which didn't resolve; found "${arrayKey}" in response`
+        detail: `Response is a root-level array \u2014 profile had "${profile.resultsPath}"`
       });
+    } else if (typeof responseData === "object" && responseData !== null) {
+      const topObj = responseData;
+      const arrayKey = Object.keys(topObj).find((k) => Array.isArray(topObj[k]));
+      if (arrayKey) {
+        resolvedResultsPath = arrayKey;
+        records = topObj[arrayKey];
+        report.autoFixed = report.autoFixed ?? {};
+        report.autoFixed.resultsPath = arrayKey;
+        checks.push({
+          name: `resultsPath auto-discovered \u2192 "${arrayKey}"`,
+          ok: true,
+          detail: `Profile had "${profile.resultsPath}" which didn't resolve; found "${arrayKey}" in response`
+        });
+      }
     }
   }
-  if (!Array.isArray(records)) {
+  if (records === null) {
     const topKeys = typeof responseData === "object" && responseData !== null ? Object.keys(responseData) : [];
     checks.push({
       name: `resultsPath "${resolvedResultsPath}" \u2192 array`,
@@ -6211,10 +6328,22 @@ async function testSyncProfile(api, profile) {
     });
     return report;
   }
+  let wrappedPrimitives = false;
+  if (records.length > 0 && typeof records[0] !== "object") {
+    const idField = profile.idField || "id";
+    const wrapped = [];
+    for (const v of records) {
+      if (typeof v === "object") continue;
+      wrapped.push({ [idField]: String(v) });
+    }
+    records = wrapped;
+    wrappedPrimitives = true;
+  }
+  const pathLabel = isRootPath(resolvedResultsPath) ? "<root>" : `"${resolvedResultsPath}"`;
   checks.push({
-    name: `resultsPath "${resolvedResultsPath}" \u2192 array`,
+    name: `resultsPath ${pathLabel} \u2192 array`,
     ok: true,
-    detail: `${records.length} records`
+    detail: wrappedPrimitives ? `${records.length} primitive records (wrapped as { ${profile.idField || "id"}: value })` : `${records.length} records`
   });
   if (records.length === 0) {
     checks.push({ name: "sample record available", ok: false, detail: "empty result set" });
@@ -8501,7 +8630,7 @@ one --agent sync sql stripe "SELECT count(*) FROM balanceTransactions"
 \`sync init\` without \`--config\` does all of this automatically:
 - **connectionKey** \u2014 auto-resolved when there's exactly one connection for the platform
 - **Pagination** \u2014 Stripe id-pagination, Notion body-cursor, HubSpot/Google token, offset, link. Inapplicable fields stripped (no nextPath for offset, no passAs for none)
-- **resultsPath** \u2014 generic keys (data, results, items) + platform-specific (model name stripped of platform prefix: attioCompanies \u2192 companies)
+- **resultsPath** \u2014 generic keys (data, results, items) + platform-specific (model name stripped of platform prefix: attioCompanies \u2192 companies). Use \`""\`, \`"$"\`, or \`"."\` for responses that return a bare array at the root (e.g. Hacker News \`/v0/topstories.json\`); primitive array elements are auto-wrapped as \`{ [idField]: value }\`.
 - **idField** \u2014 id, _id, uuid
 - **pathVars** \u2014 extracted from URL template with smart defaults (calendarId="primary", userId="me"). Internal keys (INTERNAL_SIGNING_KEY) and record-level IDs (record_id) are stripped automatically
 - **dateFilter** \u2014 updated_since, created_after, etc.
@@ -8666,7 +8795,7 @@ Fetches ALL records and deletes local rows whose IDs are no longer in the source
 |-------|----------|-------------|
 | connectionKey | yes | From \`one list\` |
 | actionId | yes | Auto-resolved by \`sync init\` |
-| resultsPath | yes | Auto-inferred or auto-discovered by \`sync test\` |
+| resultsPath | yes | Auto-inferred or auto-discovered by \`sync test\`. Use \`""\` / \`"$"\` / \`"."\` for root-array responses |
 | idField | yes | Auto-inferred or auto-discovered by \`sync test\` |
 | pagination | yes | Auto-inferred (cursor/token/offset/id/link/none) |
 | pathVars | no | Auto-extracted from URL template |
@@ -8694,7 +8823,7 @@ Fetches ALL records and deletes local rows whose IDs are no longer in the source
 .one/sync/
   profiles/{platform}_{model}.json    # sync profiles
   data/{platform}.db                  # SQLite databases (WAL mode)
-  sync_state.json                     # checkpoint tracking
+  state/{platform}/{model}.json       # per-model checkpoint tracking
   events/{platform}_{model}.jsonl     # change event logs (if onChange: "log")
   logs/{platform}.log                 # cron run logs
   locks/{platform}_{model}/           # cross-process sync locks

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@withone/cli",
-  "version": "1.37.2",
+  "version": "1.38.0",
   "description": "CLI for managing One",
   "type": "module",
   "files": [
@@ -16,7 +16,8 @@
     "build": "tsup",
     "dev": "tsup --watch",
     "start": "node bin/cli.js",
-    "typecheck": "tsc --noEmit"
+    "typecheck": "tsc --noEmit",
+    "test": "tsx --test \"src/**/*.test.ts\""
   },
   "dependencies": {
     "@clack/prompts": "^0.9.1",

package/profiles/hacker-news/topStories.json

@@ -0,0 +1,11 @@
+{
+  "description": "Hacker News top story IDs — up to 500 item IDs at /v0/topstories.json (root-array response, primitives wrapped as { id })",
+  "platform": "hacker-news",
+  "model": "topStories",
+  "connectionKey": "AUTO",
+  "actionId": "conn_mod_def::GJ3108Dwmm4::avAMAq7HQtW6PT8JhPg5vA",
+  "resultsPath": "",
+  "idField": "id",
+  "pagination": { "type": "none" },
+  "limitParam": ""
+}