@lorekit/cli 1.50.0 → 1.52.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/bin/lorekit.mjs CHANGED
@@ -94,7 +94,8 @@ ${c.bold('Commands')}
94
94
  bootstrap Apply the BYOD schema to a user-supplied Supabase database.
95
95
  Only needed when using LOREKIT_STORAGE_URL / LOREKIT_STORAGE_ANON_KEY.
96
96
  See docs/byod.md for setup instructions.
97
- migrate Relocate a LoreKit-format local store into the current layout.
97
+ migrate Relocate a LoreKit-format local store into the current layout,
98
+ or push it up to the hosted store with --to remote.
98
99
  Dry-run by default; pass --yes to apply. Idempotent.
99
100
  hook Hook engine for Claude Code / Cursor / Codex. Reads the host's
100
101
  JSON on stdin and injects memories or a retrospective nudge.
@@ -121,8 +122,8 @@ ${c.bold('Options')}
121
122
  --base <url> Dashboard base URL for deep links (link / --link; else LOREKIT_APP_URL, default https://lorekit.io)
122
123
  --threshold <0..1> Duplicate-similarity cutoff (dedupe; default 0.8)
123
124
  --from <path> Source store to migrate from (migrate)
124
- --to <tier> Migration destination tier: home | project (migrate;
125
- default routes each entry by scope)
125
+ --to <dest> Migration destination: home | project | remote (migrate;
126
+ default routes each entry by scope across the local tiers)
126
127
  --apply Apply the migration (alias of --yes) (migrate)
127
128
  -y, --yes Non-interactive / apply; never prompt
128
129
  --hooks <mode> Lifecycle hooks to wire: all | read-only | none (install)
@@ -158,6 +159,7 @@ ${c.bold('Examples')}
158
159
  npx @lorekit/cli doctor --deep
159
160
  npx @lorekit/cli migrate --from .lore # preview a rename
160
161
  npx @lorekit/cli migrate --from .lore --to project --yes
162
+ npx @lorekit/cli migrate --from .lorekit --to remote --yes # push local lore up
161
163
 
162
164
  Run ${c.cyan('lorekit <command> --help')} for command-specific options.
163
165
  `;
@@ -603,7 +605,7 @@ ${c.bold('Examples')}
603
605
  npx @lorekit/cli url --q "flaky test" --owner personal # search + ownership filter
604
606
  npx @lorekit/cli link global --tags "perf,ci" # Explorer filtered to labels
605
607
  `,
606
- migrate: `${c.bold('lorekit migrate')} — relocate a LoreKit-format local store into the current layout
608
+ migrate: `${c.bold('lorekit migrate')} — relocate a LoreKit-format local store, or push it to the hosted store
607
609
 
608
610
  ${c.bold('Usage')}
609
611
  npx @lorekit/cli migrate --from <path> [options]
@@ -613,13 +615,34 @@ Dry-run by default; pass --yes (or --apply) to write. Idempotent.
613
615
  ${c.bold('Options')}
614
616
  -d, --dir <path> Target project root (default: current directory)
615
617
  --from <path> Source store to migrate from (required)
616
- --to <tier> Destination tier: home | project (default routes by scope)
618
+ --to <dest> Destination: home | project | remote (default routes by
619
+ scope across the local tiers)
617
620
  --apply Apply the migration (alias of --yes)
618
621
  -y, --yes Apply the migration; never prompt
619
622
 
623
+ ${c.bold('--to remote')}
624
+ Pushes every entry in the source store up to the hosted store, over the
625
+ connection and token \`lorekit install\` configured (\`--endpoint\` / \`--token\`
626
+ override both). A read-only \`lk_ro_*\` token is rejected before anything is
627
+ written; an unrecognized prefix only warns and proceeds, so a self-hosted or
628
+ custom token still works.
629
+
630
+ What the hosted store does NOT take verbatim:
631
+ - archived and expired entries are skipped — a write would insert a second,
632
+ live row beside the archived one, and any TTL would re-date an expired one
633
+ - \`tags\` REPLACE the hosted row's labels, so an untagged local entry clears
634
+ them
635
+ - a creation date is honoured only when the lesson is new to the hosted
636
+ store, and an unusable one is dropped for the write instant
637
+ - \`updated\` and the \`seen_count\` tally are re-derived by the server, and a
638
+ TTL beyond 365 days is shortened
639
+ Every one of those is reported per entry, in the dry run as well as the apply.
640
+
620
641
  ${c.bold('Examples')}
621
642
  npx @lorekit/cli migrate --from .lore # preview a rename
622
643
  npx @lorekit/cli migrate --from .lore --to project --yes
644
+ npx @lorekit/cli migrate --from .lorekit --to remote # preview the push
645
+ npx @lorekit/cli migrate --from .lorekit --to remote --yes # push local lore up
623
646
  `,
624
647
  hook: `${c.bold('lorekit hook')} — hook engine for Claude Code / Cursor / Codex
625
648
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@lorekit/cli",
3
- "version": "1.50.0",
3
+ "version": "1.52.0",
4
4
  "description": "Install the LoreKit shared-memory skill and run health checks for the LoreKit MCP server.",
5
5
  "license": "MIT",
6
6
  "repository": {
package/src/mcp.mjs CHANGED
@@ -158,6 +158,30 @@ export function normalizeRunEnvironment(raw) {
158
158
  return /^[A-Za-z0-9_.\-:]+$/.test(t) ? t : null;
159
159
  }
160
160
 
161
+ /**
162
+ * The retry delay a failed response advertised, in whole seconds, or null.
163
+ *
164
+ * Prefers the JSON body's `retryAfterSeconds` over the `Retry-After` header:
165
+ * both are set by the same `tooManyRequests()` helper, and the body value is
166
+ * the number the rate-limit RPC actually returned, while the header is its
167
+ * stringified copy that an intermediary may rewrite.
168
+ *
169
+ * TOTAL over any input. `headers` is read through optional calls because a
170
+ * test double (and a hand-rolled Response-alike) may not implement the Headers
171
+ * interface, and a missing retry hint must never be able to throw on an error
172
+ * path — the caller is already handling a failure.
173
+ */
174
+ export function retryAfterFrom(data, headers) {
175
+ const raw = data?.retryAfterSeconds ?? (typeof headers?.get === 'function' ? headers.get('retry-after') : null);
176
+ if (raw == null || raw === '') return null;
177
+ const n = Number(raw);
178
+ // A `Retry-After` may also be an HTTP-date; a non-numeric value is reported
179
+ // as "no hint" so the caller falls back to its own backoff rather than
180
+ // waiting on NaN.
181
+ if (!Number.isFinite(n) || n < 0) return null;
182
+ return Math.ceil(n);
183
+ }
184
+
161
185
  export async function restFetch(baseUrl, token, path, { method = 'GET', body, timeoutMs = 10000, traceparent } = {}) {
162
186
  const controller = new AbortController();
163
187
  const timer = setTimeout(() => controller.abort(), timeoutMs);
@@ -203,6 +227,12 @@ export async function restFetch(baseUrl, token, path, { method = 'GET', body, ti
203
227
  return {
204
228
  ok: false,
205
229
  httpStatus: res.status,
230
+ // How long the server asked the caller to wait, in seconds, or null when
231
+ // it did not say. Only a 429 carries one today (`tooManyRequests` sets
232
+ // BOTH a `retryAfterSeconds` body field and the `Retry-After` header),
233
+ // but this is read on every failure so a future 503 needs no change
234
+ // here. Additive: existing callers ignore the extra key.
235
+ retryAfter: retryAfterFrom(data, res.headers),
206
236
  error: data?.error ? { message: data.error, code: data.code } : { code: res.status, message: text.slice(0, 200) || res.statusText },
207
237
  };
208
238
  }
package/src/migrate.mjs CHANGED
@@ -1,22 +1,36 @@
1
- // `lorekit migrate --from <path> [--to home|project] [--yes|--apply]`
1
+ // `lorekit migrate --from <path> [--to home|project|remote] [--yes|--apply]`
2
2
  //
3
3
  // Relocation / rename tool — NOT a persistent-memory importer. It reads a
4
4
  // LoreKit-format local store at <path> (e.g. an old `.lore/` directory, or a
5
5
  // store that was moved elsewhere) and re-writes its entries into the resolved
6
6
  // current-layout store(s), so lessons are never stranded by a rename or a move.
7
7
  //
8
+ // `--to remote` is the one destination that leaves the machine: it pushes the
9
+ // source store up to the hosted store, which is the local→remote transition a
10
+ // user who started offline and then connected a token had no bulk path for.
11
+ // Everything else about the command is unchanged by it — same dry-run default,
12
+ // same per-scope report, same idempotency — because the destination is just
13
+ // another store behind the same `getEntry`/`putEntry` pair.
14
+ //
8
15
  // Dry-run (preview) by default: it prints what would move, per scope, and
9
16
  // changes nothing. Only `--yes` (or `--apply`) mutates. Idempotent: entries are
10
17
  // upserted verbatim by scope+key, so a re-run is all NOOP.
11
18
  //
12
19
  // Out of scope: reading persistent-memory's `~/.agent-memory/<bucket>/INDEX.md`
13
20
  // + `entries/` format. This tool only understands LoreKit's own on-disk format.
21
+ // Also out of scope: the reverse direction (remote → local) and `--org <slug>`
22
+ // org-owned writes; a v1 migration always lands as the caller's personal lore.
14
23
  import fs from 'node:fs';
15
24
  import path from 'node:path';
16
- import { resolveProjectRoot } from './config.mjs';
17
- import { localStoreDirs } from './control.mjs';
18
- import { createLocalStore, createTwoTierStore } from './store/index.mjs';
25
+ import { resolveProjectRoot, tokenKind } from './config.mjs';
26
+ import { localStoreDirs, loadControl } from './control.mjs';
27
+ import { createLocalStore, createTwoTierStore, createRemoteStore } from './store/index.mjs';
19
28
  import { parseEntry } from './store/format.mjs';
29
+ import { isLive, TTL_MAX_DAYS } from './store/ttl.mjs';
30
+ import { remoteWriteLosses } from './store/remote.mjs';
31
+ import {
32
+ createPacer, withRetry, isMemoryCap, sleep, DEFAULT_CONSECUTIVE_FAILURE_LIMIT,
33
+ } from './store/rate-limit.mjs';
20
34
  import { log, heading, status, err, c } from './util.mjs';
21
35
 
22
36
  // Recursively collect every parseable LoreKit entry under a base dir. The
@@ -65,7 +79,206 @@ function sameEntry(a, b) {
65
79
  return norm(a) === norm(b);
66
80
  }
67
81
 
68
- export async function migrate(args) {
82
+ // The remote counterpart of `sameEntry`, and deliberately a DIFFERENT one —
83
+ // not weaker, but mirrored against what the hosted write actually stores.
84
+ //
85
+ // It compares only the fields a hosted write can actually change. Everything
86
+ // else the row carries is server-owned, and including any of it would make the
87
+ // command permanently non-idempotent rather than more accurate:
88
+ //
89
+ // `created_at` is honoured on INSERT only, so a row that already exists with
90
+ // a different creation date can never be made to match — every
91
+ // re-run would report UPDATE and re-push the whole store.
92
+ // `updated_at` is stamped at the write instant, so it differs by definition
93
+ // the moment the comparison runs.
94
+ // `expires_at` is recomputed from `ttl_days` at write time, so the two
95
+ // instants drift apart the moment they agree. What is
96
+ // comparable is whether the hosted expiry SATISFIES the local
97
+ // one — see `sameRemoteTtl`.
98
+ //
99
+ // A remote NOOP therefore means "the hosted lesson already says this", not
100
+ // "the two rows are byte-identical". That is the honest guarantee, and it is
101
+ // the one that makes a second `--yes` a no-op.
102
+ function sameRemoteEntry(current, entry, now = new Date()) {
103
+ if (!current || !entry) return false;
104
+ // The hosted write TRIMS `value` (`MemoryWriteSchema`), so comparing the
105
+ // untrimmed local text against the trimmed hosted one would report UPDATE
106
+ // for a padded entry on every single run and re-push it forever.
107
+ const value = (e) => (e.value == null ? '' : String(e.value)).trim();
108
+ const scalar = (e) =>
109
+ JSON.stringify({
110
+ tags: [...(e.tags || [])].sort(),
111
+ source_agent: e.source_agent ?? null,
112
+ trigger: e.trigger ?? null,
113
+ value: value(e),
114
+ });
115
+ if (scalar(current) !== scalar(entry)) return false;
116
+
117
+ // Provenance is COALESCED server-side, so a local entry that knows nothing
118
+ // about a field can never make the hosted row forget it. Comparing the two
119
+ // directly would report UPDATE forever; the honest question is whether the
120
+ // fields this entry DOES carry already match.
121
+ for (const f of ['origin_repo', 'origin_branch', 'origin_commit', 'origin_pr']) {
122
+ if (entry[f] == null) continue;
123
+ if (String(current[f] ?? '') !== String(entry[f])) return false;
124
+ }
125
+
126
+ return sameRemoteTtl(current.expires_at, entry.expires_at, now);
127
+ }
128
+
129
+ // Whether the hosted row's expiry still honours the local entry's intent.
130
+ //
131
+ // The instants cannot be compared. A push recomputes `expires_at` from
132
+ // `ttl_days` at the write instant and it then stays fixed, while the local
133
+ // intent is always measured from NOW — so the two drift apart the moment they
134
+ // agree, and any test for equality (or for "hosted >= what I would write
135
+ // today") re-pushes the entry on every run forever. An over-365-day entry is
136
+ // the worst case: its hosted row is capped at the API maximum and can never
137
+ // catch up with the local date at all.
138
+ //
139
+ // So the question is not "do these match" but "has the hosted lesson lost
140
+ // enough of its intended life to be worth rewriting":
141
+ //
142
+ // both permanent → honoured.
143
+ // one permanent, one not → different intent, re-push.
144
+ // both expiring → honoured while the hosted row still has at least
145
+ // HALF the life the local entry asks for (the ask
146
+ // itself capped at the API maximum, since that is
147
+ // the longest a write can request). A fresh push
148
+ // leaves the two equal and the entry then coasts
149
+ // for half its TTL before one re-push refreshes
150
+ // it — bounded and convergent, never a loop.
151
+ //
152
+ // The threshold is what makes it converge, and it is deliberately generous in
153
+ // both directions: a hosted row expiring in 7 days does NOT honour a 300-day
154
+ // lesson, and one expiring within the hour does NOT honour a one-day lesson,
155
+ // so a genuinely shortened TTL still migrates.
156
+ const TTL_HONOURED_FRACTION = 0.5;
157
+
158
+ function sameRemoteTtl(currentExpiry, entryExpiry, now = new Date()) {
159
+ if (!currentExpiry && !entryExpiry) return true;
160
+ if (!currentExpiry || !entryExpiry) return false;
161
+ const hosted = Date.parse(currentExpiry);
162
+ const local = Date.parse(entryExpiry);
163
+ // An unparseable value on either side is not a difference we can act on —
164
+ // `putEntry` leaves the hosted expiry alone in that case, so re-pushing
165
+ // would change nothing.
166
+ if (Number.isNaN(hosted) || Number.isNaN(local)) return true;
167
+ const at = now.getTime();
168
+ // What a write could actually ask for, from now: the local intent, capped at
169
+ // the API's maximum.
170
+ const asked = Math.min(local - at, TTL_MAX_DAYS * 86_400_000);
171
+ if (asked <= 0) return true; // the local entry is expiring anyway
172
+ return hosted - at >= asked * TTL_HONOURED_FRACTION;
173
+ }
174
+
175
+ // The global connection flags, folded into the environment the resolver reads.
176
+ // A COPY of the shim `doctor.mjs` and `mcp-server.mjs` use (kept local for the
177
+ // same reason theirs are: `resolveControl` is a pure resolver that takes an
178
+ // env object, and threading flags through it is each command's own business).
179
+ function withOverrides(args) {
180
+ const env = { ...process.env };
181
+ if (args.endpoint) env.LOREKIT_MCP_URL = args.endpoint;
182
+ if (args.token) env.LOREKIT_TOKEN = args.token;
183
+ if (args.store) env.LOREKIT_STORE = args.store;
184
+ return env;
185
+ }
186
+
187
+ // Name the entries a remote write altered, then tally them. A bare count tells
188
+ // a user something was changed without telling them what — and these are
189
+ // silent changes, so they are the ones most worth naming.
190
+ function reportAltered(keys, apply, what) {
191
+ if (!keys.length) return;
192
+ for (const key of keys.slice(0, ALTERED_LIST_CAP)) status('warn', key, what);
193
+ if (keys.length > ALTERED_LIST_CAP) {
194
+ log(` ${c.dim(`… and ${keys.length - ALTERED_LIST_CAP} more`)}`);
195
+ }
196
+ log(` ${keys.length} entr${keys.length === 1 ? 'y' : 'ies'}: ${apply ? '' : 'would have '}${what}.`);
197
+ }
198
+
199
+ // How many altered entries to name before summarising the rest. A migration
200
+ // can carry thousands; naming every one would bury the summary it belongs to.
201
+ const ALTERED_LIST_CAP = 10;
202
+
203
+ // Resolve the hosted store to push to, or the reason we cannot.
204
+ //
205
+ // Every check here is a PREFLIGHT: it runs once, before the first entry, so a
206
+ // misconfigured run fails in one line instead of halfway through a push with
207
+ // an unknown amount already written.
208
+ //
209
+ // Returns `{ error }` when the destination is unusable, else
210
+ // `{ store, endpoint, warnings, classify }` — where `classify` is false when
211
+ // the token cannot READ, so the caller must not try to.
212
+ export function resolveRemoteDestination(control, args = {}) {
213
+ const denied = (control.denies || []).find((d) => d.mode === 'remote');
214
+ if (denied) {
215
+ return { error: `remote mode is denied by ${denied.source} — a migration cannot override a deny` };
216
+ }
217
+
218
+ // The global `-e/--endpoint` and `-t/--token` outrank the resolved
219
+ // connection. `withOverrides` alone is not enough for these two:
220
+ // `resolveProjectConnection` reads `process.env` directly rather than the
221
+ // injected env, so a flag that never touches `process.env` would be silently
222
+ // ignored — which is exactly what happened before.
223
+ const resolved = control.connection || {};
224
+ const endpoint = (typeof args.endpoint === 'string' && args.endpoint.trim()) || resolved.endpoint || null;
225
+ const token = (typeof args.token === 'string' && args.token.trim()) || resolved.token || null;
226
+ const conn = {
227
+ endpoint,
228
+ token,
229
+ usable: Boolean(endpoint && token && !String(endpoint).includes('<project-ref>')),
230
+ };
231
+ if (!conn.usable) {
232
+ return {
233
+ error: 'no usable remote connection is configured.\n'
234
+ + ` Run ${c.cyan('lorekit install --endpoint <url> --token lk_rw_...')}, or set `
235
+ + 'LOREKIT_MCP_URL + LOREKIT_TOKEN.',
236
+ };
237
+ }
238
+
239
+ // The token's PREFIX is a local, offline claim about its permissions, which
240
+ // is exactly what a preflight needs: a read-only token is rejected before a
241
+ // single request rather than 403-ing on the first write, mid-run.
242
+ const kind = tokenKind(conn.token);
243
+ if (kind === 'read-only') {
244
+ return {
245
+ error: 'the configured token is read-only (lk_ro_*) — a migration to remote writes.\n'
246
+ + ` Create a read+write token (${c.cyan('lk_rw_*')}) in the dashboard and re-run.`,
247
+ };
248
+ }
249
+
250
+ const warnings = [];
251
+ // Whether the destination can be READ to classify ADD / UPDATE / NOOP. A
252
+ // write-only token's reads are denied outright, so asking would 403 every
253
+ // entry — see `classify: false` below.
254
+ let classify = true;
255
+ if (kind === 'write-only') {
256
+ // A write-only token is legitimate here — the writes will succeed — but the
257
+ // classifying READ will 403, so every entry looks new. Say so up front
258
+ // rather than presenting a plan that silently overstates the work.
259
+ classify = false;
260
+ warnings.push(
261
+ 'token is write-only (lk_wo_*) — reads are denied, so the destination is not read at all: '
262
+ + 'every entry is reported as "add" and pushed. The writes are still idempotent server-side.',
263
+ );
264
+ } else if (kind === 'unknown') {
265
+ warnings.push(
266
+ 'token has an unrecognized prefix (expected lk_rw_* / lk_ro_* / lk_wo_*) — proceeding anyway; '
267
+ + 'a token without write permission will fail on the first write.',
268
+ );
269
+ }
270
+
271
+ return {
272
+ store: createRemoteStore({ endpoint: conn.endpoint, token: conn.token }),
273
+ endpoint: conn.endpoint,
274
+ warnings,
275
+ classify,
276
+ };
277
+ }
278
+
279
+ // `options` is a test seam, not a user-facing surface: `sleepFn` lets the
280
+ // rate-limit tests exercise the backoff without actually waiting out a window.
281
+ export async function migrate(args, options = {}) {
69
282
  const root = resolveProjectRoot(args.dir);
70
283
 
71
284
  const from = typeof args.from === 'string' ? args.from : null;
@@ -80,8 +293,8 @@ export async function migrate(args) {
80
293
  }
81
294
 
82
295
  const to = typeof args.to === 'string' ? args.to.toLowerCase() : null;
83
- if (to && to !== 'home' && to !== 'project') {
84
- err(`${c.red('migrate:')} --to must be "home" or "project"`);
296
+ if (to && to !== 'home' && to !== 'project' && to !== 'remote') {
297
+ err(`${c.red('migrate:')} --to must be "home", "project" or "remote"`);
85
298
  return 1;
86
299
  }
87
300
  const apply = Boolean(args.apply || args.yes);
@@ -93,7 +306,24 @@ export async function migrate(args) {
93
306
  // repo/branch → project when opted-in, else home).
94
307
  let targetFor;
95
308
  let destLabel;
96
- if (to === 'home') {
309
+ const remote = to === 'remote';
310
+ let remoteWarnings = [];
311
+ let classifyRemote = true;
312
+ if (remote) {
313
+ // `withOverrides` so the global `-t/--token` and `-e/--endpoint` reach the
314
+ // resolver, exactly as `doctor` and the stdio MCP server do. Without it a
315
+ // user who passed both on the command line still failed the preflight.
316
+ const control = loadControl(root, { env: withOverrides(args) });
317
+ const dest = resolveRemoteDestination(control, args);
318
+ if (dest.error) {
319
+ err(`${c.red('migrate:')} ${dest.error}`);
320
+ return 1;
321
+ }
322
+ remoteWarnings = dest.warnings;
323
+ classifyRemote = dest.classify !== false;
324
+ targetFor = () => dest.store;
325
+ destLabel = `remote (${dest.endpoint})`;
326
+ } else if (to === 'home') {
97
327
  const store = createLocalStore(dirs.home);
98
328
  targetFor = () => store;
99
329
  destLabel = `home (${dirs.home})`;
@@ -108,30 +338,148 @@ export async function migrate(args) {
108
338
  destLabel = 'resolved layout (global→home, repo/branch→project-if-opted-in)';
109
339
  }
110
340
 
111
- const entries = collectEntries(src);
341
+ // ONE clock for the run: the TTL comparison, the lossiness preview and the
342
+ // conversion inside `putEntry` must not disagree about what "now" is.
343
+ const now = options.now instanceof Date ? options.now : new Date();
344
+
345
+ const collected = collectEntries(src);
346
+ // A remote destination cannot represent an archived or an expired entry
347
+ // (see `RemoteStore.putEntry`), so they are filtered here and counted rather
348
+ // than pushed and silently revived. A local destination keeps taking them —
349
+ // `LocalStore.putEntry` writes them verbatim, hidden state included, which is
350
+ // the whole point of a relocation.
351
+ const entries = remote ? collected.filter((e) => isLive(e)) : collected;
352
+ const skipped = collected.length - entries.length;
112
353
 
113
354
  heading('LoreKit migrate');
114
355
  log(` from: ${c.dim(src)}`);
115
356
  log(` to: ${c.dim(destLabel)}`);
116
357
  log(` mode: ${apply ? c.bold('apply') : 'dry-run — pass --yes to apply'}`);
117
- log(` found: ${entries.length} entr${entries.length === 1 ? 'y' : 'ies'}\n`);
358
+ log(` found: ${collected.length} entr${collected.length === 1 ? 'y' : 'ies'}\n`);
359
+ for (const w of remoteWarnings) status('warn', 'token', w);
360
+ if (skipped > 0) {
361
+ status('warn', 'skipped', `${skipped} archived or expired entr${skipped === 1 ? 'y' : 'ies'} — not representable remotely`);
362
+ }
363
+
364
+ // Stay under the hosted 120 req/min ceiling proactively, and survive a 429
365
+ // reactively. Both are no-ops for a local destination, which issues no
366
+ // requests at all — `pace()` never fills its window and `withRetry` never
367
+ // sees a rate-limited result.
368
+ let pacedOnce = false;
369
+ const pace = remote
370
+ ? createPacer({
371
+ // The pacer's wait is deliberately NOT `options.sleepFn`. That seam
372
+ // exists so a retry test does not wait out a backoff, and a sleep that
373
+ // returns without the clock advancing would spin this window forever.
374
+ // Pacing is bounded by real time, always.
375
+ sleepFn: async (ms) => {
376
+ // Say it once. A few-thousand-entry store otherwise sits silent for
377
+ // minutes after the header and looks hung.
378
+ if (!pacedOnce) {
379
+ pacedOnce = true;
380
+ status('info', 'pacing', 'staying under the hosted rate limit — this run will take longer');
381
+ }
382
+ return sleep(ms);
383
+ },
384
+ ...(options.maxPerWindow ? { maxPerWindow: options.maxPerWindow } : {}),
385
+ ...(options.windowMs ? { windowMs: options.windowMs } : {}),
386
+ })
387
+ : async () => {};
388
+ const call = async (fn) => {
389
+ if (!remote) return fn();
390
+ // `pace()` is INSIDE the retried function, so every attempt is counted
391
+ // against the window. Pacing only the first one would leave the ceiling
392
+ // unenforced during exactly the 429 episodes it exists to prevent.
393
+ return withRetry(async () => { await pace(); return fn(); }, {
394
+ sleepFn: options.sleepFn,
395
+ onRetry: ({ attempt, delayMs }) =>
396
+ status('warn', 'rate limit', `retrying in ${Math.round(delayMs / 1000)}s (attempt ${attempt})`),
397
+ });
398
+ };
118
399
 
119
400
  const totals = { add: 0, update: 0, noop: 0 };
120
401
  const byScope = new Map();
402
+ let capped = null;
403
+ let failed = 0;
404
+ // Named, not merely counted — the same courtesy the failure list gets, so a
405
+ // user knows WHICH lessons were altered rather than how many.
406
+ const clamped = [];
407
+ const redated = [];
408
+ // A blip is worth retrying; an outage is not. Once this many entries fail
409
+ // back to back the destination is not having a bad moment, it is down — and
410
+ // continuing means every remaining entry pays the full retry budget before
411
+ // failing anyway. Reset by any entry that succeeds.
412
+ const failureLimit = options.consecutiveFailureLimit ?? DEFAULT_CONSECUTIVE_FAILURE_LIMIT;
413
+ let consecutiveFailures = 0;
414
+ let abandoned = false;
121
415
  for (const entry of entries) {
122
416
  const store = targetFor(entry.scope);
123
- const current = store.getEntry({ scope: entry.scope, key: entry.key });
417
+ // Awaited so the loop is destination-agnostic: LocalStore.getEntry is
418
+ // synchronous and awaiting its plain return value is a no-op, while a
419
+ // remote destination's is a REST round-trip. One code path, both stores.
420
+ //
421
+ // A remote read that FAILS throws rather than answering null (see
422
+ // `RemoteStore.getEntry`), precisely so it cannot be mistaken for "not
423
+ // there" and overwrite a hosted lesson. Report that entry and move on: one
424
+ // unreadable key must not abort a migration, and it must not be counted as
425
+ // migrated either. The local store never throws here.
426
+ let current;
427
+ try {
428
+ // A write-only token's reads are denied, so asking would 403 every entry
429
+ // and fail a run the preflight just promised would work. Skip straight
430
+ // to the write; the hosted upsert is idempotent either way.
431
+ current = classifyRemote
432
+ ? await call(() => store.getEntry({ scope: entry.scope, key: entry.key }))
433
+ : null;
434
+ } catch (e) {
435
+ failed++;
436
+ status('fail', entry.scope, `${entry.key}: ${e?.message || 'read failed'}`);
437
+ if (++consecutiveFailures >= failureLimit) { abandoned = true; break; }
438
+ continue;
439
+ }
124
440
  let verdict;
125
441
  if (!current) verdict = 'add';
126
- else if (sameEntry(current, entry)) verdict = 'noop';
442
+ else if (remote ? sameRemoteEntry(current, entry, now) : sameEntry(current, entry)) verdict = 'noop';
127
443
  else verdict = 'update';
128
444
 
445
+ // What this entry WOULD lose, computed from the entry itself so a DRY RUN
446
+ // warns about exactly what an apply would do — reporting it only on the
447
+ // write made the preview quietly less informative than the thing it
448
+ // previews. Recorded only once the outcome is known, though: a write that
449
+ // failed shortened nothing.
450
+ const losses = remote && verdict !== 'noop'
451
+ ? remoteWriteLosses(entry, now)
452
+ : { ttlClamped: false, createdAtDropped: false };
453
+ const recordLosses = () => {
454
+ if (losses.ttlClamped) clamped.push(`${entry.scope}::${entry.key}`);
455
+ if (losses.createdAtDropped) redated.push(`${entry.scope}::${entry.key}`);
456
+ };
457
+ if (!apply) recordLosses();
458
+
459
+ if (apply && verdict !== 'noop') {
460
+ const res = await call(() => store.putEntry(entry, { now }));
461
+ if (res && res.ok === false) {
462
+ // The memory cap is terminal for the whole run, not just this entry:
463
+ // every remaining write would hit the same ceiling. Stop, and report
464
+ // what did land — a partial migration the user can resume after
465
+ // archiving or upgrading is far more useful than a stack trace.
466
+ if (isMemoryCap(res)) {
467
+ capped = res.error?.message || 'memory cap reached';
468
+ break;
469
+ }
470
+ failed++;
471
+ status('fail', entry.scope, `${entry.key}: ${res.error?.message || res.networkError || 'write failed'}`);
472
+ if (++consecutiveFailures >= failureLimit) { abandoned = true; break; }
473
+ continue; // not counted as migrated — the report must not claim it
474
+ }
475
+ recordLosses(); // the write landed, so the loss actually happened
476
+ }
477
+ consecutiveFailures = 0;
478
+
129
479
  totals[verdict]++;
130
480
  const s = byScope.get(entry.scope) || { add: 0, update: 0, noop: 0 };
131
481
  s[verdict]++;
132
482
  byScope.set(entry.scope, s);
133
-
134
- if (apply && verdict !== 'noop') await store.putEntry(entry);
135
483
  }
136
484
 
137
485
  for (const [scope, s] of byScope) {
@@ -144,6 +492,38 @@ export async function migrate(args) {
144
492
  ` ${apply ? 'migrated' : 'would migrate'} ${moved} entr${moved === 1 ? 'y' : 'ies'} ` +
145
493
  `(${totals.add} new, ${totals.update} updated), ${totals.noop} unchanged.`,
146
494
  );
495
+ if (skipped > 0) log(` ${skipped} skipped (archived or expired).`);
496
+ reportAltered(clamped, apply, 'TTL shortened to the hosted maximum of 365 days');
497
+ reportAltered(redated, apply, 'unusable created date dropped — the server stamps the write instant');
498
+
499
+ // A cap can fire after an entry has already failed, so the failure count is
500
+ // reported on BOTH exits — it used to be lost whenever the cap won the race.
501
+ const reportFailures = () => {
502
+ if (failed > 0) {
503
+ // "failed", not "failed to write" — an entry lands here from an
504
+ // unreadable classification as well as from a rejected write.
505
+ err(`${c.red('migrate:')} ${failed} entr${failed === 1 ? 'y' : 'ies'} failed (listed above).`);
506
+ }
507
+ };
508
+ if (abandoned) {
509
+ err(`\n${c.red('migrate:')} stopped after ${failureLimit} consecutive failures — the destination looks unavailable.`);
510
+ log(` ${moved} entr${moved === 1 ? 'y' : 'ies'} migrated before it gave up.`);
511
+ log(` ${c.dim('Re-run when it is healthy — the migration resumes where it stopped.')}`);
512
+ reportFailures();
513
+ return 1;
514
+ }
515
+ if (capped) {
516
+ err(`\n${c.red('migrate:')} ${capped}`);
517
+ log(` ${moved} entr${moved === 1 ? 'y' : 'ies'} migrated before the cap was reached.`);
518
+ log(` ${c.dim('Archive unused memories or raise the plan limit, then re-run — the migration resumes where it stopped.')}`);
519
+ reportFailures();
520
+ return 1;
521
+ }
522
+ if (failed > 0) {
523
+ err('');
524
+ reportFailures();
525
+ return 1;
526
+ }
147
527
  if (!apply && moved > 0) log(` ${c.dim('Re-run with --yes to apply.')}`);
148
528
  return 0;
149
529
  }
@@ -29,7 +29,7 @@ export const FIELDS = [
29
29
  // columns: a file written before this existed simply decodes it as absent.
30
30
  'expires_at',
31
31
  // Recurrence — how many times this lesson has been written, mirroring the
32
- // hosted `memories.seen_count` column (migration 00058) so an offline store
32
+ // hosted `memories.seen_count` column (migration 00059) so an offline store
33
33
  // carries the same salience signal a remote one does. Appended like the
34
34
  // columns above: a file written before this existed decodes it as absent,
35
35
  // which the read projection reports as 0 rather than inventing a count.
@@ -134,7 +134,7 @@ class LocalStore {
134
134
  archived_at: null,
135
135
  expires_at,
136
136
  // Recurrence, counted the way the hosted `memory_write` RPC counts it
137
- // (migration 00058): a write against a key this store already holds IS
137
+ // (migration 00059): a write against a key this store already holds IS
138
138
  // the next sighting. `seenCountOf` floors an absent/hand-edited value to
139
139
  // 0, so a file written before this column existed resumes at 1 on its
140
140
  // next write rather than throwing or restarting the tally at 2.
@@ -0,0 +1,149 @@
1
+ // Client-side rate-limit handling for a bulk REST push (`migrate --to remote`).
2
+ //
3
+ // The hosted API allows 120 requests/min/user (docs/limits.md), enforced by a
4
+ // Postgres fixed-window counter. A migration is the one CLI flow that can
5
+ // exceed that on its own: it issues up to two requests per entry (a read to
6
+ // classify, a write to apply), so a few hundred lessons blow the window in
7
+ // seconds. Two independent guards, because either alone is wrong:
8
+ //
9
+ // `createPacer` — PROACTIVE. Keeps the client under a self-imposed ceiling
10
+ // below the server's, so a normal run never trips the limit
11
+ // and never pays a retry.
12
+ // `withRetry` — REACTIVE. The ceiling is a guess (the limit is per USER,
13
+ // not per process — a concurrent agent shares it, and a
14
+ // per-user override can move it), so a 429 must still be
15
+ // survivable rather than a failed migration.
16
+ //
17
+ // Both take their clock and sleep as parameters, so the tests are instant and
18
+ // deterministic rather than actually waiting out a window.
19
+ //
20
+ // Zero-dependency.
21
+
22
+ export const DEFAULT_MAX_PER_WINDOW = 100; // under the 120/min server limit
23
+ export const WINDOW_MS = 60_000;
24
+ export const DEFAULT_MAX_ATTEMPTS = 5;
25
+ // After this many entries fail in a row, stop retrying and let the caller
26
+ // abort. Retrying is worth it for a blip; a systematic outage just turns a
27
+ // 2,000-entry push into hours of backoff before failing anyway.
28
+ export const DEFAULT_CONSECUTIVE_FAILURE_LIMIT = 5;
29
+ export const DEFAULT_RETRY_DELAY_MS = 1_000;
30
+ export const MAX_RETRY_DELAY_MS = 60_000;
31
+
32
+ export function sleep(ms) {
33
+ return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)));
34
+ }
35
+
36
+ /**
37
+ * A sliding-window pacer: `await pace()` before each request.
38
+ *
39
+ * Returns immediately while fewer than `maxPerWindow` requests were issued in
40
+ * the last `windowMs`; otherwise waits exactly until the oldest one falls out
41
+ * of the window. A small migration therefore runs at full speed and pays
42
+ * nothing for this, and a large one self-throttles instead of being throttled.
43
+ */
44
+ export function createPacer({
45
+ maxPerWindow = DEFAULT_MAX_PER_WINDOW,
46
+ windowMs = WINDOW_MS,
47
+ now = () => Date.now(),
48
+ sleepFn = sleep,
49
+ } = {}) {
50
+ const issued = [];
51
+ return async function pace() {
52
+ for (;;) {
53
+ const cutoff = now() - windowMs;
54
+ while (issued.length && issued[0] <= cutoff) issued.shift();
55
+ if (issued.length < maxPerWindow) {
56
+ issued.push(now());
57
+ return;
58
+ }
59
+ // Wait out the oldest request, then re-check: the clock moved, so the
60
+ // window has to be re-evaluated rather than assumed clear.
61
+ await sleepFn(issued[0] + windowMs - now() + 1);
62
+ }
63
+ };
64
+ }
65
+
66
+ /**
67
+ * Whether a store result is a retryable rate-limit rejection.
68
+ *
69
+ * `memory_cap` is the reason this is a function and not a status comparison:
70
+ * the memory-cap trigger (LK001) is ALSO translated to HTTP 429
71
+ * (supabase/functions/_shared/api/errors.ts), and it is terminal — retrying it
72
+ * just burns the user's rate budget on a write that can never succeed.
73
+ */
74
+ export function isRateLimited(res) {
75
+ return Boolean(res && res.httpStatus === 429 && res.error?.code !== 'memory_cap');
76
+ }
77
+
78
+ /**
79
+ * Whether a store result is worth trying again at all.
80
+ *
81
+ * Wider than `isRateLimited` on purpose. This module exists for a BULK push of
82
+ * potentially thousands of requests, which is precisely the shape of run where
83
+ * a single transient 5xx or a dropped connection is likeliest — and failing
84
+ * one entry out of two thousand for a blip the next attempt would sail through
85
+ * is a bad trade. So a 5xx and a transport error retry too.
86
+ *
87
+ * What does NOT retry is anything the server has decided: a 4xx is a rejection
88
+ * (bad scope, denied permission), and `memory_cap` is terminal even though it
89
+ * arrives as a 429. Retrying either just burns the user's rate budget.
90
+ */
91
+ export function isRetryable(res) {
92
+ if (!res || isMemoryCap(res)) return false;
93
+ if (res.networkError) return true;
94
+ if (isRateLimited(res)) return true;
95
+ return typeof res.httpStatus === 'number' && res.httpStatus >= 500;
96
+ }
97
+
98
+ /** Whether a store result is the terminal memory-cap rejection. */
99
+ export function isMemoryCap(res) {
100
+ return Boolean(res && res.error?.code === 'memory_cap');
101
+ }
102
+
103
+ /**
104
+ * Run `fn` and retry it while it comes back retryable (see `isRetryable`).
105
+ *
106
+ * Honours the server's own `Retry-After` when it sent one — it knows when its
107
+ * window rolls over and the client does not — and falls back to exponential
108
+ * backoff otherwise. Returns the last result once the attempts are spent, so
109
+ * the caller reports a real error rather than a synthesised one; `onRetry` is
110
+ * for progress output.
111
+ */
112
+ export async function withRetry(fn, {
113
+ maxAttempts = DEFAULT_MAX_ATTEMPTS,
114
+ baseDelayMs = DEFAULT_RETRY_DELAY_MS,
115
+ sleepFn = sleep,
116
+ onRetry = null,
117
+ } = {}) {
118
+ let res;
119
+ for (let attempt = 1; attempt <= maxAttempts; attempt++) {
120
+ // A read reports a rate limit by THROWING (`RemoteStore.getEntry` refuses
121
+ // to let a failed read look like an absence), so the rejection has to be
122
+ // classified here too — otherwise only writes would ever be retried and a
123
+ // 429 on the classifying read would fail the entry outright. Anything that
124
+ // is not a rate limit propagates untouched.
125
+ try {
126
+ res = await fn();
127
+ } catch (e) {
128
+ if (!isRetryable(e?.result) || attempt === maxAttempts) throw e;
129
+ await sleepFn(retryDelay(e.result, attempt, baseDelayMs, onRetry));
130
+ continue;
131
+ }
132
+ if (!isRetryable(res) || attempt === maxAttempts) return res;
133
+ await sleepFn(retryDelay(res, attempt, baseDelayMs, onRetry));
134
+ }
135
+ return res;
136
+ }
137
+
138
+ // How long to wait before the next attempt: the server's own hint when it sent
139
+ // one — it knows when its window rolls over and the client does not — else
140
+ // exponential backoff. Shared by the resolved and the thrown path so the two
141
+ // cannot drift.
142
+ function retryDelay(res, attempt, baseDelayMs, onRetry) {
143
+ const hinted = Number(res?.retryAfter);
144
+ const delayMs = Number.isFinite(hinted) && hinted > 0
145
+ ? Math.min(hinted * 1000, MAX_RETRY_DELAY_MS)
146
+ : Math.min(baseDelayMs * 2 ** (attempt - 1), MAX_RETRY_DELAY_MS);
147
+ if (onRetry) onRetry({ attempt, delayMs });
148
+ return delayMs;
149
+ }
@@ -21,6 +21,7 @@
21
21
  import { restFetch, mcpToRestBase } from '../mcp.mjs';
22
22
  import { getActiveTraceparent } from '../telemetry.mjs';
23
23
  import { withReadFields } from './entry-fields.mjs';
24
+ import { normalizeCreatedAt } from './created-at.mjs';
24
25
 
25
26
  // Drop undefined/null args so JSON payloads stay tidy.
26
27
  function stripUndefined(obj) {
@@ -29,6 +30,90 @@ function stripUndefined(obj) {
29
30
  return out;
30
31
  }
31
32
 
33
+ /**
34
+ * A read that could not be answered — a transport failure or a non-2xx status,
35
+ * as opposed to "the lesson is not there".
36
+ *
37
+ * Exported so a caller can tell it from a programming error and degrade
38
+ * per-entry (report this one, keep going) instead of aborting a whole run.
39
+ * `result` carries the raw store envelope for the message the caller shows.
40
+ */
41
+ export class StoreReadError extends Error {
42
+ constructor(message, result) {
43
+ super(message);
44
+ this.name = 'StoreReadError';
45
+ this.result = result;
46
+ }
47
+ }
48
+
49
+ /**
50
+ * What a remote write of `entry` would lose, without performing it.
51
+ *
52
+ * Exported because a DRY RUN has to be able to say the same thing the apply
53
+ * will: a preview that omits "this entry's expiry will be shortened" is a
54
+ * preview of a different operation. `putEntry` calls it too, so the two
55
+ * cannot drift.
56
+ *
57
+ * `ttlClamped` the entry's TTL exceeds the API's 365-day maximum and
58
+ * will land shortened.
59
+ * `createdAtDropped` the entry's `created` is unusable (unparseable or
60
+ * future-dated), so the override is dropped and the
61
+ * server stamps the write instant instead.
62
+ *
63
+ * Total over any input: a malformed entry reports no losses rather than
64
+ * throwing, because this runs on the preview path where an exception would
65
+ * cost the user the whole plan.
66
+ */
67
+ export function remoteWriteLosses(entry, now = new Date()) {
68
+ const exact = remoteTtlDaysExact(entry?.expires_at, now);
69
+ return {
70
+ ttlClamped: typeof exact === 'number' && exact > 365,
71
+ createdAtDropped: Boolean(entry?.created) && safeCreatedAt(entry?.created, now) === null,
72
+ };
73
+ }
74
+
75
+ // `normalizeCreatedAt`, but an invalid value yields null instead of throwing.
76
+ function safeCreatedAt(created, now) {
77
+ try {
78
+ return normalizeCreatedAt(created ?? null, now);
79
+ } catch {
80
+ return null;
81
+ }
82
+ }
83
+
84
+ // An absolute `expires_at` expressed as the hosted write's relative `ttl_days`.
85
+ //
86
+ // Three outcomes, and the third exists because "no expiry" and "I cannot tell"
87
+ // must not collapse into one answer:
88
+ //
89
+ // `undefined` no expiry — the caller states that positively with
90
+ // `clear_ttl: true`, so a permanent lesson stops being expiring.
91
+ // `'expired'` already elapsed; the caller must refuse (see `putEntry`).
92
+ // `'unknown'` an unparseable value. The caller then sends NEITHER TTL field,
93
+ // leaving the RPC on its `'keep'` branch, because the safe
94
+ // reading of a corrupt frontmatter field is "do not touch the
95
+ // expiry" — the same fail-safe posture as `isExpired`. Treating
96
+ // it as no expiry would let one bad character wipe a live remote
97
+ // TTL.
98
+ //
99
+ // else the remaining WHOLE days, clamped to the schema's 1–365.
100
+ function remoteTtlDays(expiresAt, now = new Date()) {
101
+ const exact = remoteTtlDaysExact(expiresAt, now);
102
+ if (typeof exact !== 'number') return exact;
103
+ return Math.min(365, Math.max(1, exact));
104
+ }
105
+
106
+ // The same conversion WITHOUT the 1–365 clamp, so a caller can see that the
107
+ // clamp bound and report the loss. Same three non-numeric outcomes.
108
+ function remoteTtlDaysExact(expiresAt, now = new Date()) {
109
+ if (!expiresAt) return undefined;
110
+ const ms = Date.parse(expiresAt);
111
+ if (Number.isNaN(ms)) return 'unknown';
112
+ const remaining = ms - now.getTime();
113
+ if (remaining <= 0) return 'expired';
114
+ return Math.ceil(remaining / 86_400_000);
115
+ }
116
+
32
117
  export function createRemoteStore({ endpoint, token } = {}) {
33
118
  return new RemoteStore(endpoint, token);
34
119
  }
@@ -151,7 +236,24 @@ class RemoteStore {
151
236
  // scope+key is unique, so one row is all there can be — don't pull the default page of 50.
152
237
  p.set('limit', '1');
153
238
  const res = await this._rest(`/memories?${p}`);
154
- if (!res.ok) return { ok: false, error: res.error, networkError: res.networkError };
239
+ // `unusable` is passed through: `_rest` short-circuits an unconfigured
240
+ // store with that flag and NOTHING else, so a caller that drops it is left
241
+ // with a failure carrying no error and no networkError — a blank failure it
242
+ // can only report generically. Additive; every existing caller branches on
243
+ // `ok` and ignores the extra key.
244
+ if (!res.ok) {
245
+ return {
246
+ ok: false,
247
+ error: res.error ?? null,
248
+ // Carried for the same reason `write` carries them: a read can be
249
+ // rate-limited too, and a caller that retries needs to tell a 429 it
250
+ // should wait out from one it must not.
251
+ httpStatus: res.httpStatus ?? null,
252
+ retryAfter: res.retryAfter ?? null,
253
+ networkError: res.networkError ?? null,
254
+ unusable: res.unusable ?? false,
255
+ };
256
+ }
155
257
  const entries = res.data?.entries ?? [];
156
258
  // Same projection as list/search — a single read must not answer with a
157
259
  // different shape than the listing the caller found the key in.
@@ -181,7 +283,214 @@ class RemoteStore {
181
283
  if (origin_commit !== undefined) body.origin_commit = origin_commit;
182
284
  if (origin_pr !== undefined) body.origin_pr = origin_pr;
183
285
  const res = await this._rest('/memories', { method: 'POST', body });
184
- return { ok: res.ok, error: res.error, networkError: res.networkError };
286
+ // `httpStatus` and `retryAfter` are passed through so a caller can tell the
287
+ // two 429s apart and honour the server's own backoff. They are DIFFERENT
288
+ // failures wearing one status code: `code: 'rate_limited'` is transient and
289
+ // must be retried, `code: 'memory_cap'` is terminal (translateDbError maps
290
+ // the LK001 cap trigger to 429 as well) and must not be. Additive — the
291
+ // existing `{ ok, error, networkError }` keys are unchanged.
292
+ // Every field is coalesced, not just the retry hint: a caller comparing
293
+ // `httpStatus` must not get `null` from a refusal and `undefined` from the
294
+ // network-error or `unusable` branch, which is the exact split the shape
295
+ // exists to remove.
296
+ return {
297
+ ok: res.ok,
298
+ error: res.error ?? null,
299
+ httpStatus: res.httpStatus ?? null,
300
+ retryAfter: res.retryAfter ?? null,
301
+ networkError: res.networkError ?? null,
302
+ // `_rest` short-circuits an unconfigured store with `{ ok:false,
303
+ // unusable:true }` and no error at all, so without this the caller sees
304
+ // a failure with every field null and no reason. Passed through the way
305
+ // `listScopes` already does.
306
+ unusable: res.unusable ?? false,
307
+ };
308
+ }
309
+
310
+ // ── Migrate-destination parity with LocalStore ────────────────────────────
311
+ //
312
+ // `migrate` classifies each source entry ADD / UPDATE / NOOP with a read and
313
+ // then upserts it, against whatever store it was handed. LocalStore answers
314
+ // that with `getEntry` + `putEntry`; these are the remote halves, so the
315
+ // migrate loop stays ONE code path instead of branching per destination.
316
+ //
317
+ // The local pair is lossless (`putEntry` writes every field verbatim,
318
+ // archived rows included). The remote pair CANNOT be, because the hosted
319
+ // write is an RPC with a fixed parameter list, not a file write:
320
+ //
321
+ // preserved scope, key, source_agent, trigger, and `created` —
322
+ // sent as `created_at`, which memory_write honours on INSERT
323
+ // only, so a migrated lesson keeps its original creation date
324
+ // and its ranking recency with it. `value` survives too but is
325
+ // TRIMMED: `MemoryWriteSchema` applies `.transform(s =>
326
+ // s.trim())`, so surrounding whitespace does not make the trip.
327
+ // re-stamped `updated` — the server sets it to the write instant. There is
328
+ // no parameter for it, and inventing one would let a client
329
+ // backdate an edit it did not make.
330
+ // derived `seen_count` — the RPC owns the tally (migration 00059: a
331
+ // write against an existing key IS the next sighting). A lesson
332
+ // the hosted store has never seen lands at 1; one it already
333
+ // holds lands at ITS count plus one, not the local one. Either
334
+ // way the local history does not transfer.
335
+ // converted `expires_at` → `ttl_days`, the remaining whole days, clamped
336
+ // to the schema's 1–365 (a longer-lived TTL is clamped, not
337
+ // dropped — the alternative is silently making it permanent).
338
+ // A clamp IS lossy, so it is reported: the result carries
339
+ // `ttlClamped: true` and the caller can list the entry as
340
+ // shortened rather than leaving the user to discover it.
341
+ // A PERMANENT entry sends `clear_ttl: true` rather than simply
342
+ // omitting `ttl_days`: omission is the RPC's `'keep'` branch
343
+ // (migration 00031), which leaves an existing remote
344
+ // `expires_at` in place, so a permanent local lesson would
345
+ // land on an expiring remote row and still die.
346
+ // authoritative
347
+ // `tags`. The conflict clause is `tags = excluded.tags`, so the
348
+ // source entry's list REPLACES the hosted one — an untagged
349
+ // local entry sends `[]` and clears whatever labels the hosted
350
+ // row carried. That is what a verbatim upsert means here (the
351
+ // local file is the thing being migrated), but it is the one
352
+ // field where "verbatim" can remove hosted data, so it is
353
+ // called out rather than filed under preserved.
354
+ // sticky `origin_*`. The RPC's conflict clause coalesces provenance
355
+ // (`coalesce(excluded.origin_repo, memories.origin_repo)`, and
356
+ // likewise for branch/commit/pr) so a write that does not know
357
+ // a field cannot erase what an earlier one recorded. A migrated
358
+ // entry with NO provenance therefore leaves whatever the hosted
359
+ // row already had; it cannot clear it, by design, and there is
360
+ // no parameter that would. `source_agent` and `trigger` are NOT
361
+ // coalesced (`= excluded.*`), so an absent one still CLEARS the
362
+ // hosted value — not because a null is sent (`stripUndefined`
363
+ // drops nulls before the request) but because the REST handler
364
+ // substitutes `?? null` for the missing field and the RPC
365
+ // writes that. Omitted and null are the same instruction here,
366
+ // which is the opposite of what they mean for `origin_*`. `kind` and `host` are coalesced the same
367
+ // way, and this store never sends them at all — the server
368
+ // infers both from the `loop::` tag (`resolveKindHost`), which
369
+ // the tags carry, so a migrated lesson classifies itself.
370
+ //
371
+ // Two states have no remote representation at all and are REFUSED rather
372
+ // than silently rewritten, because writing them would resurrect a lesson the
373
+ // user retired: an archived entry (every conflict predicate on `memories` is
374
+ // partial on `archived_at is null`, so the hosted write does not revive the
375
+ // archived row — it INSERTS a second, live one beside it, leaving the store
376
+ // with both) and
377
+ // an already-expired one (any `ttl_days` re-dates it into the future). Both
378
+ // come back as `{ ok:false, unsupported }` so the caller can report them as
379
+ // skipped. `migrate` does NOT filter them today — adding that filter is the
380
+ // job of the PR that makes a remote destination reachable — so until then
381
+ // this refusal is the only thing standing between an archived lesson and
382
+ // resurrection, which is why it lives in the store and not in the caller.
383
+
384
+ // Raw lookup by scope+key, mirroring `LocalStore.getEntry` — the entry or
385
+ // null. The ROW is each store's own: this answers a REST `MemoryEntry`
386
+ // (`created_at`/`updated_at`) and the local one answers parsed frontmatter
387
+ // (`created`/`updated`), both through `withReadFields`. That is deliberate —
388
+ // the pair exists so a caller can ASK either store whether a key is there,
389
+ // not so it can compare two rows field-by-field without knowing which store
390
+ // produced them. A caller that compares has to speak the destination's
391
+ // spelling; that is what the remote comparison in the migrate loop does. LocalStore's is synchronous and this one cannot be, so callers must
392
+ // `await` it; awaiting the local store's plain return value is a no-op.
393
+ //
394
+ // One semantic difference the caller has to know about: LocalStore.getEntry
395
+ // sees archived rows and this cannot — `GET /memories` filters them out — so
396
+ // a remote destination classifies an archived counterpart as ADD, and the
397
+ // write then lands as a NEW live row beside the archived one (the conflict
398
+ // predicates are partial on `archived_at is null`). That is what the hosted
399
+ // `memory_write` does for any write against an archived key, not a
400
+ // migrate-specific quirk — and it is why `putEntry` refuses an archived
401
+ // source entry outright rather than relying on this classification.
402
+ //
403
+ // A FAILED read THROWS rather than answering null. Null is the answer to "no
404
+ // such lesson", and a caller that classifies ADD / UPDATE / NOOP acts on it:
405
+ // returning null for a transient 500 or a dropped connection would quietly
406
+ // reclassify an existing hosted lesson as new and overwrite it. The local
407
+ // store never throws here (a file read that fails is genuinely a miss), so
408
+ // this widens the contract only where the failure mode exists.
409
+ async getEntry({ scope, key } = {}) {
410
+ const res = await this.read({ scope, key });
411
+ if (!res.ok) {
412
+ // An unconfigured store carries no error at all, so name that case
413
+ // rather than reporting the generic failure text for it.
414
+ const reason = res.unusable
415
+ ? 'the remote store is not configured (missing endpoint or token)'
416
+ : (res.networkError || res.error?.message || 'read failed');
417
+ throw new StoreReadError(`remote read failed for ${scope}::${key}: ${reason}`, res);
418
+ }
419
+ return res.entry ?? null;
420
+ }
421
+
422
+ // Upsert one entry, as close to verbatim as the hosted write allows. See the
423
+ // fidelity table above for exactly which fields survive.
424
+ //
425
+ // EVERY branch answers with the SAME key set — `write`'s
426
+ // `{ ok, error, httpStatus, retryAfter, networkError, unusable }` plus
427
+ // Unlike `LocalStore.putEntry`, no `entry` comes back: the hosted write
428
+ // returns an id and derives the rest server-side, so echoing the request as
429
+ // if it were the stored row would be a fabrication. A caller that needs the
430
+ // stored row reads it back.
431
+ //
432
+ // `unsupported` (the refusal reason, else null), `ttlClamped` (the entry
433
+ // landed with a shortened life) and `createdAtDropped` (its `created` was
434
+ // unusable, so the server stamped the write instant instead). The last two
435
+ // report what HAPPENED, so both are false when the write did not succeed. A refusal fills the transport fields
436
+ // with null rather than omitting them, so a caller reading any one key never
437
+ // gets a value from one branch and `undefined` from another.
438
+ async putEntry(entry = {}, { now = new Date() } = {}) {
439
+ const refuse = (unsupported, message) => ({
440
+ ok: false,
441
+ unsupported,
442
+ // Carries a `code` like every real failure, so a caller can branch on
443
+ // one field instead of matching prose.
444
+ error: { message, code: 'unsupported' },
445
+ httpStatus: null,
446
+ retryAfter: null,
447
+ networkError: null,
448
+ // Every key any putEntry branch answers with, so a caller reading one
449
+ // never gets a value from one branch and `undefined` from another.
450
+ unusable: false,
451
+ ttlClamped: false,
452
+ createdAtDropped: false,
453
+ });
454
+ if (entry?.archived_at) {
455
+ return refuse('archived', 'archived entries cannot be written remotely — the hosted write would insert a second, live row beside the archived one');
456
+ }
457
+ const ttl = remoteTtlDays(entry?.expires_at, now);
458
+ if (ttl === 'expired') {
459
+ return refuse('expired', 'expired entries cannot be written remotely — any TTL would re-date them into the future');
460
+ }
461
+ // What this write will lose, decided by the same pure function a DRY RUN
462
+ // calls — so a preview cannot promise something the apply then silently
463
+ // does differently.
464
+ const { ttlClamped, createdAtDropped } = remoteWriteLosses(entry, now);
465
+ const createdAt = createdAtDropped ? undefined : (safeCreatedAt(entry?.created, now) ?? undefined);
466
+ const result = await this.write(stripUndefined({
467
+ scope: entry.scope,
468
+ key: entry.key,
469
+ value: entry.value == null ? '' : String(entry.value),
470
+ tags: Array.isArray(entry.tags) ? entry.tags : [],
471
+ source_agent: entry.source_agent,
472
+ trigger: entry.trigger,
473
+ created_at: createdAt,
474
+ // `'unknown'` sends neither field, leaving the RPC on its `'keep'`
475
+ // branch. A real TTL sends only `ttl_days`; no TTL says so explicitly
476
+ // with `clear_ttl` rather than by omission — see the fidelity note above.
477
+ ttl_days: typeof ttl === 'number' ? ttl : undefined,
478
+ clear_ttl: ttl === undefined ? true : undefined,
479
+ origin_repo: entry.origin_repo,
480
+ origin_branch: entry.origin_branch,
481
+ origin_commit: entry.origin_commit,
482
+ origin_pr: entry.origin_pr,
483
+ }));
484
+ // Always present, like every other key in this envelope — a caller must
485
+ // not have to know which branch produced the result to read it.
486
+ // Both flags describe what HAPPENED, so they are false on a write that did
487
+ // not happen — a failed request shortened nothing and re-dated nothing.
488
+ return {
489
+ ...result,
490
+ unsupported: null,
491
+ ttlClamped: Boolean(result.ok) && ttlClamped,
492
+ createdAtDropped: Boolean(result.ok) && createdAtDropped,
493
+ };
185
494
  }
186
495
 
187
496
  // Natural-key DELETE. Without `force` the server soft-archives (stamps
package/src/util.mjs CHANGED
@@ -21,12 +21,32 @@ export const sym = {
21
21
  info: useColor ? c.cyan('•') : '-',
22
22
  };
23
23
 
24
+ // Where `log` and `err` send their output. Indirected through a mutable pair
25
+ // so a test can capture a command's output WITHOUT hijacking
26
+ // `process.stdout.write` — which is not a private channel: `node --test` runs
27
+ // each file in a child process and reports results over that same stdout, so a
28
+ // global hijack silently swallows the runner's own result lines and a failing
29
+ // test can go unreported. Production behaviour is unchanged; the default
30
+ // writers are the streams themselves.
31
+ let writers = {
32
+ out: (s) => process.stdout.write(s),
33
+ err: (s) => process.stderr.write(s),
34
+ };
35
+
36
+ // Redirect `log`/`err` (and therefore `heading`/`status`, which build on them).
37
+ // Returns the restore function, so a caller cannot forget what it replaced.
38
+ export function setWriters(next = {}) {
39
+ const previous = writers;
40
+ writers = { ...writers, ...next };
41
+ return () => { writers = previous; };
42
+ }
43
+
24
44
  export function log(msg = '') {
25
- process.stdout.write(`${msg}\n`);
45
+ writers.out(`${msg}\n`);
26
46
  }
27
47
 
28
48
  export function err(msg = '') {
29
- process.stderr.write(`${msg}\n`);
49
+ writers.err(`${msg}\n`);
30
50
  }
31
51
 
32
52
  export function heading(title) {