@lanes-sh/link 0.7.2 → 0.9.0

package/src/cli/contract3-credentials.ts

@@ -0,0 +1,294 @@
+import { ConfigError, LEGACY_DATA_DIR, isRemoteWorkspace } from '#profile';
+import { C3 } from './contract3-layout.ts';
+import { createFileSecretStore } from '#secrets';
+
+/**
+ * The credential half of the contract-3 migration.
+ *
+ * Split from `contract3-data.ts` when that file outgrew the budget, on the seam
+ * the migration already had: objects move, credentials merge, and the two obey
+ * different rules. A moved object has one home and the move is reversible by
+ * moving it back. A merged credential has to survive two profiles claiming one
+ * ref, and the wrong resolution points a live connection at somebody else's
+ * account — so nothing here picks between two values, ever.
+ *
+ * The whole file is ordered around one promise: every refusal happens before
+ * the first write.
+ */
+
+/**
+ * What this migration needs to know about one profile's credentials.
+ *
+ * Assembled by `contract3.ts` from the hoist, because the two facts below are
+ * decisions that file already made and this one must not make differently.
+ */
+export interface CredentialPlan {
+  readonly profile: string;
+  /** Every connection-derived credential ref, and where it is going. */
+  readonly renames: ReadonlyMap<string, RefTarget>;
+  /** This profile's endpoint token ref, which is deliberately not migrated. */
+  readonly tokenRef: string;
+}
+
+/** What the merged credential store will hold, and what was left out of it. */
+export interface CredentialMerge {
+  /** Refs the merged store will hold, sorted. */
+  readonly refs: readonly string[];
+  /** Endpoint token refs left behind rather than merged, sorted. */
+  readonly tokens: readonly string[];
+}
+
+/** Where a credential ref is going, and which connection it belongs to after. */
+export interface RefTarget {
+  readonly to: string;
+  /** The settled `provider.id` this ref's connection became. */
+  readonly connection: string;
+}
+
+/**
+ * The credential ref of every connection the hoist touched, renamed or not.
+ *
+ * `credentialRefForConnection` derives `${app ?? provider}/${connectionId}` for
+ * every auth kind that has a per-connection credential, and `hoistConnections`
+ * renames the connection with `{ ...connection, id }` — which leaves the derived
+ * ref pointing at the old id. So a profile whose `github.main` became
+ * `github.main_2` still claimed `github/main`, and two profiles claiming one ref
+ * with two different tokens is what aborted the migration.
+ *
+ * **Unrenamed connections are in here too**, which is not redundancy: the
+ * `connection` half is what tells `readMerged` that two refs landing on one
+ * target belong to the same connection and are therefore a merge rather than a
+ * clash. Without it a single profile declaring `gmail.main` and `gmail.archive`
+ * for one mailbox — legal under contract 2, and the reason tokens are
+ * per-connection at all, since the scopes differ — was hoisted into one row and
+ * then refused with "Two profiles hold different values", naming one profile
+ * twice and prescribing a rename that cannot help.
+ *
+ * Only the `<provider>/<id>` spelling is derived here. A provider declaring
+ * `auth.app` stores under `<app>/<id>` and a row carrying an explicit
+ * `credential_ref` stores wherever it says — neither is reconstructable without
+ * the manifest, which this migration does not load. Those refs are carried
+ * across untouched, and a genuine clash between two of them is refused by name.
+ */
+export function connectionRefs(mapping: ReadonlyMap<string, string>): Map<string, RefTarget> {
+  const refs = new Map<string, RefTarget>();
+
+  for (const [from, to] of mapping) {
+    const before = from.indexOf('.');
+    const after = to.indexOf('.');
+    if (before < 0 || after < 0) continue;
+    refs.set(`${from.slice(0, before)}/${from.slice(before + 1)}`, {
+      to: `${to.slice(0, after)}/${to.slice(after + 1)}`,
+      connection: to,
+    });
+  }
+
+  return refs;
+}
+
+/**
+ * Every credential the merged store will hold, addressed as contract 3 will
+ * address it.
+ *
+ * One walk, shared by the preview and the apply, because the two disagreeing is
+ * the defect this replaces: `planCredentials` collected the union of ref *names*
+ * and never compared values, so a clash it could not see aborted
+ * `mergeCredentials` — at which point `rewriteRegistry` and `writeConnections`
+ * had already run, and the workspace was half-migrated behind a message saying
+ * nothing had been written.
+ *
+ * Read-only. Everything that can refuse, refuses here.
+ *
+ * **A workspace in a bucket has nothing to merge, and this says so rather than
+ * finding out.** `workspacePath` refuses a filesystem adapter against a remote
+ * root, so the only credential store such a workspace can declare is
+ * `gcp-secret-manager` — whose refs were never scoped by profile, and are
+ * therefore already what contract 3 wants. Without the guard the path below is
+ * built by string interpolation into `gs://bucket/data/<profile>/credentials.enc`
+ * and handed to `Bun.file`, where the failure is swallowed by the `catch` and
+ * reads exactly like a workspace with no credentials in it.
+ */
+async function readMerged(
+  root: string,
+  plans: readonly CredentialPlan[],
+): Promise<{ merged: Map<string, string>; tokens: Set<string> }> {
+  const merged = new Map<string, string>();
+  const held = new Map<string, { profile: string; connection?: string }>();
+  // One entry per endpoint-token ref, holding what each profile had under it.
+  // Decided after the walk, because whether it can be carried across depends on
+  // whether the profiles agree — which is not known until they have all been read.
+  const endpoint = new Map<string, Map<string, string>>();
+  const tokens = new Set<string>();
+
+  if (isRemoteWorkspace(root)) return { merged, tokens };
+
+  for (const plan of plans) {
+    const store = createFileSecretStore({
+      path: `${root}/${LEGACY_DATA_DIR}/${plan.profile}/credentials.enc`,
+    });
+
+    let refs: string[];
+    try {
+      refs = await store.list();
+    } catch {
+      // A store that will not open is reported by `doctor`, not here. This runs
+      // as a preview too, and must not fail on a workspace that is already
+      // broken.
+      continue;
+    }
+
+    for (const ref of refs) {
+      // The endpoint's own bearer token, which every profile keeps under the
+      // same ref because `authSchema` defaults `token_ref` to `profile/token`.
+      // Under contract 2 that was unambiguous — one store per profile. Under
+      // contract 3 there is one store, so three profiles' tokens are three
+      // values for one key, and there is no merge that means anything.
+      //
+      // Left behind rather than picked between. It is minted locally rather
+      // than granted by anybody, `ensureProfileToken` writes a fresh one the
+      // first time a command asks, and the old stores are not deleted — so the
+      // cost is re-registering a client, and no account has to be authorised
+      // again.
+      let value: string | null;
+      try {
+        value = await store.get(ref);
+      } catch {
+        continue;
+      }
+      if (value === null) continue;
+
+      // The endpoint's own bearer token, held under one ref by every profile
+      // because `authSchema` defaults `token_ref` to `profile/token`. Set aside
+      // rather than merged here; see below.
+      if (ref === plan.tokenRef) {
+        const seen = endpoint.get(ref) ?? new Map<string, string>();
+        seen.set(plan.profile, value);
+        endpoint.set(ref, seen);
+        continue;
+      }
+
+      const mapped = plan.renames.get(ref);
+      const target = mapped?.to ?? ref;
+      const first = merged.get(target);
+
+      if (first !== undefined) {
+        if (first === value) continue;
+
+        // Two refs on one target whose connections are the same connection.
+        // That is the hoist merging two rows for one account — legal under
+        // contract 2, where `gmail.main` and `gmail.archive` could hold the
+        // same mailbox at different scopes — and there is one connection now,
+        // so one token. The first row is the one that survived the hoist, so
+        // its credential is the one that belongs to it.
+        const owner = held.get(target);
+        if (mapped !== undefined && owner?.connection === mapped.connection) continue;
+
+        throw new ConfigError(
+          `Two credentials want to be at "${target}", and this migration cannot choose ` +
+            `between them.\n` +
+            `  ${owner?.profile ?? 'another profile'} and ${plan.profile} both hold one. Both ` +
+            `are real credentials, and picking either would point a connection at the wrong ` +
+            `account.\n` +
+            `  Nothing has been written. Give one of them its own credential_ref before ` +
+            `migrating.`,
+        );
+      }
+
+      merged.set(target, value);
+      held.set(target, {
+        profile: plan.profile,
+        ...(mapped === undefined ? {} : { connection: mapped.connection }),
+      });
+    }
+  }
+
+  // **A token is only left behind when the profiles disagree about it.**
+  //
+  // Under contract 2 each profile had its own store, so one ref meant one value
+  // per profile and contract 3's single store cannot hold three of them. But a
+  // workspace with one profile — or three that were registered from the same
+  // token — has nothing to choose between, and dropping it there was gratuitous:
+  // every client registered against that endpoint started getting 401s after a
+  // routine `update`, for a conflict that did not exist.
+  //
+  // Where they do disagree it is still left behind rather than picked between.
+  // It is minted locally rather than granted by anybody, `ensureProfileToken`
+  // writes a fresh one the first time a command asks, and the old stores are not
+  // deleted — so the cost is re-registering a client, and no account has to be
+  // authorised again.
+  for (const [ref, byProfile] of endpoint) {
+    const values = new Set(byProfile.values());
+    const agreed = values.size === 1 ? [...values][0] : undefined;
+
+    if (agreed === undefined || merged.has(ref)) {
+      tokens.add(ref);
+      continue;
+    }
+
+    merged.set(ref, agreed);
+    held.set(ref, { profile: [...byProfile.keys()].join(', ') });
+  }
+
+  return { merged, tokens };
+}
+
+/**
+ * What the merge will do, computed before anything is written.
+ *
+ * The report an operator confirms is therefore the real one, and a refusal
+ * leaves the workspace exactly as it was.
+ */
+export async function planCredentials(
+  root: string,
+  plans: readonly CredentialPlan[],
+): Promise<CredentialMerge> {
+  const { merged, tokens } = await readMerged(root, plans);
+  return { refs: [...merged.keys()].sort(), tokens: [...tokens].sort() };
+}
+
+/**
+ * Copy every profile's credentials into the workspace store.
+ *
+ * Written and read back before the old stores are touched, which is the whole
+ * of the safety argument: a half-finished merge that has not deleted anything is
+ * recoverable by running it again, and one that deleted first is not.
+ *
+ * The old stores are left in place regardless. They are a few kilobytes, they
+ * are the only copy of anything if this went wrong, and `doctor` names them so
+ * an operator can remove them once the endpoint has served a request.
+ *
+ * Skipped entirely for a workspace in a bucket, for the reason `readMerged`
+ * gives: its credentials are in Secret Manager under refs that were never
+ * per-profile, so there is no second store to fold in.
+ */
+export async function mergeCredentials(
+  root: string,
+  plans: readonly CredentialPlan[],
+): Promise<void> {
+  if (isRemoteWorkspace(root)) return;
+
+  const { merged } = await readMerged(root, plans);
+  const destination = createFileSecretStore({ path: `${root}/${C3.credentials()}` });
+
+  for (const [ref, value] of merged) {
+    const already = await destination.get(ref);
+    if (already !== null) {
+      // Already copied, by a run that did not get to the end. Anything else is
+      // a store that disagrees with the profiles it was built from, which is
+      // not something to overwrite silently.
+      if (already === value) continue;
+      throw new ConfigError(
+        `${C3.credentials()} already holds a different value for "${ref}" than the profile ` +
+          `stores do. Nothing has been deleted; resolve it and run this again.`,
+      );
+    }
+
+    await destination.set(ref, value);
+    if ((await destination.get(ref)) !== value) {
+      throw new ConfigError(
+        `The credential "${ref}" did not read back after being written to ` +
+          `${C3.credentials()}. Nothing has been deleted; fix the store and run this again.`,
+      );
+    }
+  }
+}

package/src/cli/contract3-data.ts

@@ -0,0 +1,262 @@
+import { ConfigError, LEGACY_DATA_DIR } from '#profile';
+import { applyMoves, assertOneObjectPerDestination, claim, type Move } from './migrate-move.ts';
+import { C3 } from './contract3-layout.ts';
+import type { BlobStore } from '#stores/blobs';
+import { CONNECTIONS_NAMESPACE, decodeSegment, objectKey } from '#stores/state';
+
+/**
+ * The half of the contract-3 migration that moves bytes rather than YAML.
+ *
+ * Split from `contract3.ts` on the seam the migration already has: that file
+ * decides *what* the new shape is, and this one carries the credentials and
+ * objects into it. Both are ordered so a crash between any two steps leaves a
+ * workspace that still opens — nothing is deleted until it has been read back.
+ */
+
+
+/**
+ * Where every object under `data/<profile>/` is going.
+ *
+ * Driven by `perProfile`, the per-profile map the hoist already built from old
+ * key to new — because the rename that matters is *this profile's*. The first
+ * version of this keyed a lookup by the hoisted (new) key and queried it with
+ * the old one, which made the resolution an unconditional no-op: provider and
+ * connection ids contain no dot, so anything the map returned already had the id
+ * being looked up. Two profiles holding `gmail.main` for different mailboxes
+ * both sent their blobs to `data/gmail/main/`, and the second one's landed in
+ * the first one's namespace.
+ *
+ * Anything that matches no rule is left exactly where it is: this moves what it
+ * understands and never deletes what it does not.
+ */
+export async function planMoves(
+  files: BlobStore,
+  profiles: readonly string[],
+  perProfile: ReadonlyMap<string, ReadonlyMap<string, string>>,
+): Promise<Move[]> {
+  const moves: Move[] = [];
+  // Destinations inside `state.kv` that an earlier profile already claimed, for
+  // the two namespaces where a second claim is a duplicate rather than a
+  // conflict. See `stateMove`.
+  const claimed = new Set<string>();
+
+  for (const profile of profiles) {
+    const mapping = perProfile.get(profile) ?? new Map<string, string>();
+    const prefix = `${LEGACY_DATA_DIR}/${profile}/`;
+
+    for (const blob of await files.list(prefix)) {
+      const rest = blob.key.slice(prefix.length);
+      const [head, ...tail] = rest.split('/');
+      if (head === undefined) continue;
+
+      // The credential store is merged rather than moved, and the old copy is
+      // deliberately left behind.
+      if (head === 'credentials.enc' || head === 'credentials.enc.key') continue;
+
+      // The instance this profile's single-instance surfaces became. Both are
+      // one store per profile in contract 2 and one per *connection* in
+      // contract 3, so two profiles' vaults are two documents — sending both to
+      // `vault('main')` orphaned the second and silently gave it the first's,
+      // which is the worst of the collisions because the wrong answer is a
+      // credential (ADR-059).
+      if (head === 'vault.enc') {
+        moves.push({ from: blob.key, to: C3.vault(instanceOf(mapping, 'vault')) });
+        continue;
+      }
+      if (head === 'vault.enc.key') {
+        moves.push({ from: blob.key, to: `${C3.vault(instanceOf(mapping, 'vault'))}.key` });
+        continue;
+      }
+      if (head === 'skills.d') {
+        moves.push({
+          from: blob.key,
+          to: `${C3.skills(instanceOf(mapping, 'skills'))}/${tail.join('/')}`,
+        });
+        continue;
+      }
+      // Under ADR-030 a manifest lived in the profile, so an operator using
+      // their own connector in two profiles holds two copies of one file. They
+      // are the same manifest; refusing byte-identical files and asking for the
+      // data directory's layout was not a useful answer.
+      if (head === 'providers.d') {
+        const move = claim(claimed, {
+          from: blob.key,
+          to: `${C3.providers()}/${tail.join('/')}`,
+        });
+        if (move !== null) moves.push(move);
+        continue;
+      }
+      // One object per event under a key already carrying the timestamp, so
+      // concatenating three profiles' logs is exactly moving them across.
+      if (head === 'audit.log') {
+        moves.push({ from: blob.key, to: `${C3.audit()}/${tail.join('/')}` });
+        continue;
+      }
+      if (head === 'state.kv') {
+        const move = stateMove(blob.key, tail, mapping, claimed);
+        if (move !== null) moves.push(move);
+        continue;
+      }
+
+      // Otherwise it is `<provider>/<connection>/...`, the namespace every
+      // provider's blobs are scoped into.
+      const connection = tail[0];
+      if (connection === undefined) continue;
+
+      // This profile's old key, through this profile's mapping.
+      const settled = mapping.get(`${head}.${connection}`);
+      const id = settled === undefined ? connection : (settled.split('.')[1] ?? connection);
+      moves.push({ from: blob.key, to: `${LEGACY_DATA_DIR}/${head}/${id}/${tail.slice(1).join('/')}` });
+    }
+  }
+
+  assertOneObjectPerDestination(moves);
+  return moves;
+}
+
+
+/**
+ * Where one object under a profile's `state.kv` is going.
+ *
+ * `state.kv` used to be moved verbatim, on the grounds that its records "already
+ * carry the profile". They do not. `connections.v1` is keyed on
+ * `<provider>.<id>` — precisely what the hoist renames — so every profile's
+ * owner layer wrote `connections.v1/vault.main`, and three profiles aimed three
+ * objects at one key. That is the collision `assertOneObjectPerDestination`
+ * called unreachable, and it was reachable from any workspace with two profiles
+ * in it.
+ *
+ * Keys are decoded rather than pattern-matched: on disk both halves are
+ * percent-encoded per segment (`connections%2Ev1/vault%2Emain.json`), and
+ * respelling that here would be a second copy of an encoding that must have one.
+ *
+ * `null` means leave it where it is — nothing is deleted by not moving it, and
+ * `doctor` names what is left.
+ */
+function stateMove(
+  from: string,
+  tail: readonly string[],
+  mapping: ReadonlyMap<string, string>,
+  claimed: Set<string>,
+): Move | null {
+  const here = `${C3.state()}/${tail.join('/')}`;
+  const leaf = tail[tail.length - 1];
+
+  // Not an object this module wrote: moved as it is, and still held to the
+  // one-object-per-destination rule, because an unrecognised collision is a
+  // thing to refuse rather than to resolve by guessing.
+  if (tail.length < 2 || leaf === undefined || !leaf.endsWith('.json')) {
+    return { from, to: here };
+  }
+
+  // `decodeSegment` is `decodeURIComponent`, which throws `URIError` on a stray
+  // `%`. Aborting somebody's whole migration on `URI error`, naming no file, is
+  // not an answer — an undecodable key is one this does not understand, and
+  // those are moved as they are.
+  let segments: string[];
+  let key: string;
+  try {
+    segments = tail.slice(0, -1).map(decodeSegment);
+    key = decodeSegment(leaf.slice(0, -'.json'.length));
+  } catch {
+    return { from, to: here };
+  }
+  const namespace = segments.join('/');
+
+  // Keyed on the provider id, not a connection, so two profiles on the same
+  // vendor hold two entries under one key. `open.ts` treats a miss and a corrupt
+  // entry alike as "not discovered yet" and `connect` refreshes it: first wins.
+  if (namespace === 'discovery') {
+    return claim(claimed, { from, to: here });
+  }
+
+  // `<provider>/<connection>` — a provider's own state, keyed on the connection
+  // exactly as its blobs are. Left out of the first pass, so two profiles
+  // holding one provider's fixed-name object (`dav`'s home, `bunq`'s session)
+  // still aimed at one key.
+  if (segments.length === 2) {
+    const [provider, connection] = segments as [string, string];
+    const settled = mapping.get(`${provider}.${connection}`);
+    if (settled === undefined) return { from, to: here };
+
+    const id = settled.slice(settled.indexOf('.') + 1);
+    return claim(claimed, { from, to: `${C3.state()}/${objectKey(`${provider}/${id}`, key)}` });
+  }
+
+  if (namespace !== CONNECTIONS_NAMESPACE) return { from, to: here };
+
+  // This profile's old key, through this profile's mapping — the same lookup
+  // the provider-blob branch does, and wrong in the same way if it is skipped.
+  const settled = mapping.get(key);
+
+  // Orphaned under contract 2 already — no row, and `hoistConnections` reads
+  // rows — so hoisting it would manufacture a connection nothing grants, and its
+  // key can collide with a rename that is real.
+  if (settled === undefined) return null;
+
+  const to = `${C3.state()}/${objectKey(CONNECTIONS_NAMESPACE, settled)}`;
+
+  // Two profiles on one account merge into one row, so both hold a record for
+  // it. One connection; the second differs only in when it was written.
+  if (settled === key) return claim(claimed, { from, to });
+
+  const dot = settled.indexOf('.');
+  const provider = settled.slice(0, dot);
+  const id = settled.slice(dot + 1);
+  return claim(claimed, { from, to, rewrite: (data) => retarget(data, provider, id) });
+}
+
+
+/**
+ * A connection record, told what it is now called.
+ *
+ * Spread rather than assigned field by field, so key order and anything a later
+ * version added survive. Bytes that are not a JSON object are returned
+ * untouched: refusing to move what would not parse strands it under a profile
+ * directory nothing reads.
+ */
+function retarget(data: Uint8Array, provider: string, id: string): Uint8Array {
+  let record: unknown;
+  try {
+    record = JSON.parse(new TextDecoder().decode(data));
+  } catch {
+    return data;
+  }
+
+  if (record === null || typeof record !== 'object' || Array.isArray(record)) return data;
+
+  return new TextEncoder().encode(
+    JSON.stringify({ ...(record as Record<string, unknown>), provider, id }),
+  );
+}
+
+/**
+ * Copy, verify, then delete. In that order, per object.
+ *
+ * A move that deleted first would lose an object on any failure, and these are
+ * the owner's notes, tasks and entities.
+ *
+ * A destination that already holds *different* bytes is a bug rather than a case
+ * to handle, now that the hoist gives every profile's owner layer its own
+ * instance: two sets of notes can no longer be aimed at one key. It used to be
+ * skipped silently, which is how work's vault came to be orphaned while `moved`
+ * reported it as moved. `assertOneObjectPerDestination` refuses that while this
+ * is still a plan, and the check here catches what a plan cannot see.
+ *
+ * **The same bytes at the destination is the interrupted move, and it finishes
+ * it.** Copy-then-delete has a window between the two, and this migration now
+ * runs against buckets — where the window is a network round trip rather than a
+ * syscall, and an interruption is something that happens rather than something
+ * to reason about. Refusing there would have meant a workspace that could not be
+ * migrated by running the migration again, which is the one recovery this file
+ * promises.
+ */
+/** Which instance of a single-instance surface this profile's store became. */
+function instanceOf(mapping: ReadonlyMap<string, string>, provider: string): string {
+  for (const [from, to] of mapping) {
+    if (from.startsWith(`${provider}.`)) return to.split('.')[1] ?? 'main';
+  }
+  return 'main';
+}
+
+

package/src/cli/contract3-layout.ts

@@ -0,0 +1,46 @@
+import { LEGACY_DATA_DIR, LEGACY_WORKSPACE_FILE } from '#profile';
+
+/**
+ * Where contract 3 put things, frozen.
+ *
+ * `#profile`'s `layout` describes where things live *now*. A migration lives
+ * between two layouts and must not ask it: `contract3.ts` migrates to this one
+ * and `contract4.ts` migrates away from it, so both need it spelled somewhere
+ * that will not move when the live one does. Asking `layout` made the contract-3
+ * migration write a contract-4 tree, skipping the step contract 4 was about to
+ * take and leaving nothing for it to find.
+ *
+ * One module rather than a copy in each, for the reason `layout.ts` exists: the
+ * credential path was spelled in three files and they have to agree forever.
+ */
+/**
+ * The owner layer's ids through contract 3, before the `lanes_` prefix.
+ *
+ * Spelled here for the reason every other path in this file is: a migration
+ * reads the shape it is migrating *from*. `RESERVED_PROVIDER_IDS` is the live
+ * list and is `lanes_memory` now, so asking it whether a contract-2 row is
+ * owner-layer answers no for every one of them.
+ */
+export const C3_OWNER_PROVIDERS: readonly string[] = [
+  'memory',
+  'tasks',
+  'assets',
+  'skills',
+  'vault',
+  'setup',
+  'identity',
+  'entities',
+];
+
+export const C3 = {
+  /** The registry, before it became `workspaces.yaml`. */
+  workspace: LEGACY_WORKSPACE_FILE,
+  /** A profile's declaration, before it moved inside its own directory. */
+  profile: (profile: string): string => `profiles/${profile}.yaml`,
+  credentials: (): string => `${LEGACY_DATA_DIR}/credentials.enc`,
+  state: (): string => `${LEGACY_DATA_DIR}/state.kv`,
+  audit: (): string => `${LEGACY_DATA_DIR}/audit.log`,
+  providers: (): string => `${LEGACY_DATA_DIR}/providers.d`,
+  vault: (connection: string): string => `${LEGACY_DATA_DIR}/vault.d/${connection}.enc`,
+  skills: (connection: string): string => `${LEGACY_DATA_DIR}/skills.d/${connection}`,
+} as const;