@withone/cli 1.52.3 → 1.53.0

package/README.md

@@ -423,8 +423,11 @@ one sync run stripe --since 90d
 one mem sync run stripe                              # identical (alias)
 # Profiles with `enrich` run a second detail pass. It enriches each record ONCE;
 # the list pass thereafter merges rather than replaces, so the enriched payload
-# survives (reported as `memPreserved`). `--full-refresh` does not re-enrich —
-# delete .one/sync/data/<platform>.db to force that.
+# survives (reported as `memPreserved`). `--full-refresh` reconciles deletions,
+# it does not refresh detail content. To refresh detail:
+one sync run gmail --re-enrich                       # re-fetch every detail endpoint
+# ...or set `enrich.invalidateOn` in the profile (e.g. "historyId", "updated_at")
+# so only records that actually changed upstream are re-enriched, automatically.
 
 # Query + search (reads from memory)
 one sync query stripe/balanceTransactions --where "status=available" --limit 20

package/dist/index.js

@@ -5672,6 +5672,52 @@ function findEnrichedIds(db, model, idField, config2, records, tableCreated) {
   }
   return enriched;
 }
+var ENRICH_FINGERPRINT_COLUMN = "_enrich_fp";
+function ensureColumn(db, safeTable, column, type) {
+  const cols = db.prepare(`PRAGMA table_info("${safeTable}")`).all();
+  if (!cols.some((c) => c.name === column)) {
+    db.exec(`ALTER TABLE "${safeTable}" ADD COLUMN "${column}" ${type}`);
+  }
+}
+function clearEnrichmentStamps(db, model, timestampField = "_enriched_at") {
+  const safeTable = model.replace(/[^a-zA-Z0-9_]/g, "_");
+  const exists = db.prepare(
+    `SELECT name FROM sqlite_master WHERE type='table' AND name = ?`
+  ).get(safeTable);
+  if (!exists) return 0;
+  const cols = db.prepare(`PRAGMA table_info("${safeTable}")`).all();
+  if (!cols.some((c) => c.name === timestampField)) return 0;
+  const result = db.prepare(
+    `UPDATE "${safeTable}" SET "${timestampField}" = NULL WHERE "${timestampField}" IS NOT NULL`
+  ).run();
+  return result.changes;
+}
+function invalidateStaleEnrichments(db, model, config2) {
+  const fpField = config2.invalidateOn;
+  if (!fpField) return 0;
+  const tsField = config2.timestampField ?? "_enriched_at";
+  const safeTable = model.replace(/[^a-zA-Z0-9_]/g, "_");
+  const exists = db.prepare(
+    `SELECT name FROM sqlite_master WHERE type='table' AND name = ?`
+  ).get(safeTable);
+  if (!exists) return 0;
+  const cols = new Set(
+    db.prepare(`PRAGMA table_info("${safeTable}")`).all().map((c) => c.name)
+  );
+  if (!cols.has(fpField)) return 0;
+  if (!cols.has(tsField)) return 0;
+  if (!cols.has(ENRICH_FINGERPRINT_COLUMN)) {
+    ensureColumn(db, safeTable, ENRICH_FINGERPRINT_COLUMN, "TEXT");
+    return 0;
+  }
+  const result = db.prepare(
+    `UPDATE "${safeTable}" SET "${tsField}" = NULL
+      WHERE "${tsField}" IS NOT NULL
+        AND "${ENRICH_FINGERPRINT_COLUMN}" IS NOT NULL
+        AND "${ENRICH_FINGERPRINT_COLUMN}" IS NOT CAST("${fpField}" AS TEXT)`
+  ).run();
+  return result.changes;
+}
 async function enrichPhase(api, db, config2, model, idField, connectionKey, platform, ctx = {}) {
   const startTime = Date.now();
   const tsField = config2.timestampField ?? "_enriched_at";
@@ -5740,6 +5786,10 @@ async function enrichPhase(api, db, config2, model, idField, connectionKey, plat
         }
         const merged = config2.merge !== false ? deepMerge(row, enrichedData) : { ...enrichedData, [idField]: id };
         merged[tsField] = now;
+        if (config2.invalidateOn) {
+          const fp = merged[config2.invalidateOn] ?? row[config2.invalidateOn];
+          merged[ENRICH_FINGERPRINT_COLUMN] = fp == null ? null : String(fp);
+        }
         pending.push({ merged, id });
       } else if (result.status === "fulfilled" && result.value === null) {
         rateLimited++;
@@ -5803,7 +5853,7 @@ async function enrichPhase(api, db, config2, model, idField, connectionKey, plat
         hookEvents.push({ type: "update", platform, model, record: merged, timestamp: now });
       }
     }
-    if (ctx.profile && memoryBatch.length > 0) {
+    if (ctx.profile && ctx.writeToMemory !== false && memoryBatch.length > 0) {
       try {
         await writePageToMemory(ctx.profile, memoryBatch);
       } catch (err) {
@@ -6458,6 +6508,19 @@ async function syncModel(api, profile, options) {
     }
     let enrichResult = null;
     if (profile.enrich && db && tableCreated && !options.dryRun) {
+      if (options.reEnrich) {
+        const cleared = clearEnrichmentStamps(db, model, profile.enrich.timestampField);
+        if (cleared > 0 && !isAgentMode()) {
+          console.log(`    Re-enriching ${cleared} record(s) (--re-enrich)`);
+        }
+      } else {
+        const invalidated = invalidateStaleEnrichments(db, model, profile.enrich);
+        if (invalidated > 0 && !isAgentMode()) {
+          console.log(
+            `    ${invalidated} record(s) changed upstream (${profile.enrich.invalidateOn}) \u2014 re-enriching those`
+          );
+        }
+      }
       enrichResult = await enrichPhase(
         api,
         db,
@@ -6476,7 +6539,9 @@ async function syncModel(api, profile, options) {
           // Pass the full profile so enrich can mirror merged rows into the
           // memory store. Without this, memory holds only the pre-enrich
           // list payload — see enrich.ts:memoryBatch writeback.
-          profile
+          profile,
+          // Phase 1 honours --no-memory; phase 2 used to ignore it. (#174)
+          writeToMemory: options.toMemory !== false
         }
       );
       enrichedTotal = enrichResult.enriched;
@@ -8596,7 +8661,7 @@ function registerSyncSubcommands(sync) {
   sync.command("suggest-searchable <platform/model>").description("Rank candidate memory.searchable paths by signal density (for profiles with memory.embed: true)").option("--limit <n>", "Top N paths to return (default 15)").action(async (platformModel, options) => {
     await syncSuggestSearchableCommand(platformModel, options);
   });
-  sync.command("run <platform>").description("Run sync for a platform (syncs all configured models, or specify --models)").option("--models <models>", "Comma-separated list of models to sync").option("--since <duration>", "Sync records since duration (e.g. 90d, 30d, 7d) or date").option("--force", "Ignore existing sync state and start fresh").option("--max-pages <n>", "Maximum number of pages to fetch").option("--dry-run", "Fetch first page only, show results without persisting").option("--full-refresh", "Fetch ALL records and delete local rows no longer in the source (handles deletions)").option("--no-memory", "Skip the unified memory dual-write (default: memory is always written)").option("--embed", "Embed synced rows under the configured model, regardless of profile.memory.embed", false).option("--no-embed", "Skip embedding even if the profile opts in").option("--to-memory", "(deprecated \u2014 memory is now always written; flag kept for back-compat)").action(async (platform, options) => {
+  sync.command("run <platform>").description("Run sync for a platform (syncs all configured models, or specify --models)").option("--models <models>", "Comma-separated list of models to sync").option("--since <duration>", "Sync records since duration (e.g. 90d, 30d, 7d) or date").option("--force", "Ignore existing sync state and start fresh").option("--max-pages <n>", "Maximum number of pages to fetch").option("--dry-run", "Fetch first page only, show results without persisting").option("--full-refresh", "Fetch ALL records and delete local rows no longer in the source (handles deletions)").option("--re-enrich", 'Re-fetch the detail endpoint for every already-enriched record (profiles with an "enrich" block). Costs one detail call per record \u2014 for routine freshness set enrich.invalidateOn in the profile instead').option("--no-memory", "Skip the unified memory dual-write (default: memory is always written)").option("--embed", "Embed synced rows under the configured model, regardless of profile.memory.embed", false).option("--no-embed", "Skip embedding even if the profile opts in").option("--to-memory", "(deprecated \u2014 memory is now always written; flag kept for back-compat)").action(async (platform, options) => {
     await syncRunCommand(platform, {
       models: options.models?.split(",").map((m) => m.trim()),
       since: options.since,
@@ -8604,6 +8669,7 @@ function registerSyncSubcommands(sync) {
       maxPages: options.maxPages ? parseInt(options.maxPages, 10) : void 0,
       dryRun: options.dryRun,
       fullRefresh: options.fullRefresh,
+      reEnrich: options.reEnrich,
       // Commander inverts `--no-memory` / `--no-embed` into options.X === false.
       toMemory: options.memory !== false,
       // Only override when the flag was actually passed — leaving
@@ -10740,6 +10806,7 @@ When a list endpoint returns lightweight records (e.g. just IDs), add an \`enric
   "enrich": {
     "actionId": "<get-message-action-id>",
     "pathVars": { "messageId": "{{id}}" },
+    "invalidateOn": "historyId",
     "concurrency": 3,
     "delayMs": 200
   }
@@ -10747,7 +10814,8 @@ When a list endpoint returns lightweight records (e.g. just IDs), add an \`enric
 \`\`\`
 
 - \`pathVars\` / \`queryParams\` / \`body\` support \`{{field}}\` interpolation from the list record
-- \`concurrency\` controls parallel detail requests per page (default: 3, lower = safer for rate limits)
+- \`invalidateOn\` names a list field that acts as a change fingerprint, so records whose detail went stale are re-enriched automatically (see "Re-enriching" below). Omit it to keep enrich-exactly-once behaviour
+- \`concurrency\` controls parallel detail requests per page (default: 5, lower = safer for rate limits)
 - \`delayMs\` is the pause between batches (default: 200ms)
 - \`resultsPath\` extracts a sub-object from the detail response before merging
 - \`merge: false\` replaces the record entirely instead of deep-merging
@@ -10803,7 +10871,19 @@ So for a row the mirror reports as already enriched, Phase 1 writes **non-author
 
 The write still happens, so \`--embed\`, profile edits, and the store's un-archive self-heal all keep working on enriched rows. The one thing given up: a field that disappears from the *list* shape upstream no longer disappears from an enriched record, because a merge cannot delete. \`sync run\` reports the count as \`memPreserved\`.
 
-**Known gap:** \`--full-refresh\` does **not** clear \`_enriched_at\`, so it does not re-enrich \u2014 and there is currently no way to re-enrich a record whose upstream detail content changed after its first enrichment. Delete the mirror (\`.one/sync/data/<platform>.db\`) to force a full re-enrich.
+**Re-enriching.** \`--full-refresh\` re-pulls the list but deliberately does **not** clear \`_enriched_at\` \u2014 it reconciles deletions, it is not a detail refresh. There are two ways to refresh detail content:
+
+- **\`enrich.invalidateOn\` (preferred, automatic).** Name a list-endpoint field that acts as a change fingerprint \u2014 \`historyId\` for Gmail threads, \`updated_at\` for Fathom meetings. The value seen at enrich time is recorded, and on the next sync any record whose fingerprint has moved is re-enriched. Records that did not change upstream cost nothing.
+
+  \`\`\`json
+  { "enrich": { "actionId": "...", "invalidateOn": "historyId" } }
+  \`\`\`
+
+  Additive by design: rows enriched before a fingerprint existed are never auto-invalidated (that would re-enrich the whole table on the first run after upgrading) \u2014 they pick one up the next time they enrich. Profiles with no sensible fingerprint field simply omit it and keep enrich-exactly-once behaviour.
+
+- **\`one sync run <platform> --re-enrich\` (manual escape hatch).** Clears every enrichment stamp and re-fetches all detail endpoints. Costs one detail call per record, so it is opt-in per run and never implied by \`--full-refresh\`. Use it when the detail *shape* changed, or for profiles with no fingerprint field.
+
+Deleting the mirror (\`.one/sync/data/<platform>.db\`) is no longer necessary.
 
 ## Cross-Platform Identity
 
@@ -10909,7 +10989,7 @@ Every \`sync X\` command is also exposed as \`mem sync X\` \u2014 same handlers,
 | \`sync init <plat> <model>\` | Create/patch profile (seeds from built-in, auto-tests) |
 | \`sync test <plat>/<model>\` | Validate profile. \`--show-searchable\` previews embedded text across 5 samples with per-path hit rates |
 | \`sync suggest-searchable <plat>/<model>\` | Rank candidate \`memory.searchable\` paths by signal density; emits paste-ready config |
-| \`sync run <platform>\` | Sync data (\`--full-refresh\`, \`--since\`, \`--dry-run\`, \`--no-memory\`) |
+| \`sync run <platform>\` | Sync data (\`--full-refresh\`, \`--since\`, \`--dry-run\`, \`--no-memory\`, \`--re-enrich\`) |
 | \`sync query <plat>/<model>\` | Query memory with \`--where\` (dotted paths), \`--after/before\` |
 | \`sync schema <plat>/<model>\` | Inspect the JSON structure of synced records (field paths, types, examples) \u2014 run before writing \`--where\` / query paths |
 | \`sync search "<query>"\` | Hybrid FTS + semantic across all synced data |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@withone/cli",
-  "version": "1.52.3",
+  "version": "1.53.0",
   "description": "CLI for managing One",
   "type": "module",
   "files": [

package/skills/one/SKILL.md

@@ -318,7 +318,7 @@ Without declared paths, the default walker concatenates every string in the reco
 
 Both are queryable with `one --agent mem find-by-key <prefix>:<value>`.
 
-**Enriching profiles** (`gmail/gmailThreads`, `fathom/meetings`) sync in two phases: a list pass, then a detail pass that fetches full bodies/transcripts. Two things follow. Enrichment happens **once per record** — phase 2 only visits rows it has never enriched, and `--full-refresh` does *not* reset that, so re-running a sync will not refresh detail content (delete `.one/sync/data/<platform>.db` to force it). And the list pass never overwrites an enriched record: `data` merges rather than replaces, and `searchable_text` / `identity_keys[]` are left alone. `sync run` reports these as `memPreserved`. So on an enriching profile, a record whose upstream *detail* changed will look stale until the mirror is cleared — that is expected, not a sync failure.
+**Enriching profiles** (`gmail/gmailThreads`, `fathom/meetings`) sync in two phases: a list pass, then a detail pass that fetches full bodies/transcripts. Two things follow. Enrichment happens **once per record by default** — phase 2 only visits rows it has never enriched, and `--full-refresh` does *not* reset that (it reconciles deletions, it is not a detail refresh). To refresh detail content, either set `enrich.invalidateOn` in the profile to a list field that moves when the detail changes (`historyId`, `updated_at`) so only genuinely-changed records re-enrich automatically, or run `one sync run <platform> --re-enrich` to re-fetch every detail endpoint. And the list pass never overwrites an enriched record: `data` merges rather than replaces, and `searchable_text` / `identity_keys[]` are left alone. `sync run` reports these as `memPreserved`. So on an enriching profile without `invalidateOn`, a record whose upstream *detail* changed will look stale until you re-enrich — that is expected, not a sync failure.
 
 **Advanced features** (enrich, transform, exclude, hooks, --full-refresh, alternative backends, embedding tuning): run `one guide memory` or `one guide sync` for the full reference.