@lanes-sh/link 0.8.0 → 0.9.1

package/src/cli/contract3-data.ts

@@ -1,119 +1,18 @@
-import { ConfigError, DATA_DIR, isRemoteWorkspace, layout } from '#profile';
-import { createFileSecretStore } from '#secrets';
+import { ConfigError, LEGACY_DATA_DIR } from '#profile';
+import { applyMoves, assertOneObjectPerDestination, claim, type Move } from './migrate-move.ts';
+import { C3 } from './contract3-layout.ts';
 import type { BlobStore } from '#stores/blobs';
+import { CONNECTIONS_NAMESPACE, decodeSegment, objectKey } from '#stores/state';
 
 /**
  * The half of the contract-3 migration that moves bytes rather than YAML.
  *
  * Split from `contract3.ts` on the seam the migration already has: that file
  * decides *what* the new shape is, and this one carries the credentials and
- * objects into it. Both halves are ordered so a crash between any two steps
- * leaves a workspace that still opens, and the rule that makes that true lives
- * here — nothing is deleted until what replaced it has been read back.
+ * objects into it. Both are ordered so a crash between any two steps leaves a
+ * workspace that still opens — nothing is deleted until it has been read back.
  */
 
-export interface Move {
-  readonly from: string;
-  readonly to: string;
-}
-
-/** `gmail.main` — how a connection is addressed in every file after this. */
-function keyOf(connection: { provider: string; id: string }): string {
-  return `${connection.provider}.${connection.id}`;
-}
-
-/**
- * Which credential refs the merged store will hold.
- *
- * Read-only, and it runs before anything is written so the report an operator
- * confirms is the real one. A ref present in two profiles' stores with two
- * different values is the one thing this cannot resolve, and it is reported
- * rather than merged: both are real credentials, and picking either would point
- * a connection at the wrong account's token.
- *
- * **A workspace in a bucket has nothing to merge, and this says so rather than
- * finding out.** `workspacePath` refuses a filesystem adapter against a remote
- * root, so the only credential store such a workspace can declare is
- * `gcp-secret-manager` — whose refs were never scoped by profile, and are
- * therefore already what contract 3 wants. Without the guard the path below is
- * built by string interpolation into `gs://bucket/data/<profile>/credentials.enc`
- * and handed to `Bun.file`, where the failure is swallowed by the `catch` and
- * reads exactly like a workspace with no credentials in it.
- */
-export async function planCredentials(root: string, profiles: readonly string[]): Promise<string[]> {
-  if (isRemoteWorkspace(root)) return [];
-
-  const refs = new Set<string>();
-
-  for (const profile of profiles) {
-    const store = createFileSecretStore({ path: `${root}/${DATA_DIR}/${profile}/credentials.enc` });
-    try {
-      for (const ref of await store.list()) refs.add(ref);
-    } catch {
-      // A store that will not open is reported by `doctor`, not here. This is a
-      // preview and must not fail on a workspace that is already broken.
-    }
-  }
-
-  return [...refs].sort();
-}
-
-/**
- * Copy every profile's credentials into the workspace store.
- *
- * Written and read back before the old stores are touched, which is the whole
- * of the safety argument: a half-finished merge that has not deleted anything is
- * recoverable by running it again, and one that deleted first is not.
- *
- * The old stores are left in place regardless. They are a few kilobytes, they
- * are the only copy of anything if this went wrong, and `doctor` names them so
- * an operator can remove them once the endpoint has served a request.
- *
- * Skipped entirely for a workspace in a bucket, for the reason `planCredentials`
- * gives: its credentials are in Secret Manager under refs that were never
- * per-profile, so there is no second store to fold in.
- */
-export async function mergeCredentials(root: string, profiles: readonly string[]): Promise<void> {
-  if (isRemoteWorkspace(root)) return;
-
-  const destination = createFileSecretStore({ path: `${root}/${layout.credentials()}` });
-
-  for (const profile of profiles) {
-    const source = createFileSecretStore({ path: `${root}/${DATA_DIR}/${profile}/credentials.enc` });
-
-    let refs: string[];
-    try {
-      refs = await source.list();
-    } catch {
-      continue;
-    }
-
-    for (const ref of refs) {
-      const value = await source.get(ref);
-      if (value === null) continue;
-
-      const held = await destination.get(ref);
-      if (held !== null) {
-        if (held === value) continue;
-        throw new ConfigError(
-          `Two profiles hold different values for the credential "${ref}", and this migration ` +
-            `cannot choose between them.\n` +
-            `  Both are real credentials for different accounts, and picking either would point a ` +
-            `connection at the wrong one.\n` +
-            `  Rename one connection before migrating, so its credential ref differs.`,
-        );
-      }
-
-      await destination.set(ref, value);
-      if ((await destination.get(ref)) !== value) {
-        throw new ConfigError(
-          `The credential "${ref}" did not read back after being written to ` +
-            `${layout.credentials()}. Nothing has been deleted; fix the store and run this again.`,
-        );
-      }
-    }
-  }
-}
 
 /**
  * Where every object under `data/<profile>/` is going.
@@ -136,10 +35,14 @@ export async function planMoves(
   perProfile: ReadonlyMap<string, ReadonlyMap<string, string>>,
 ): Promise<Move[]> {
   const moves: Move[] = [];
+  // Destinations inside `state.kv` that an earlier profile already claimed, for
+  // the two namespaces where a second claim is a duplicate rather than a
+  // conflict. See `stateMove`.
+  const claimed = new Set<string>();
 
   for (const profile of profiles) {
     const mapping = perProfile.get(profile) ?? new Map<string, string>();
-    const prefix = `${DATA_DIR}/${profile}/`;
+    const prefix = `${LEGACY_DATA_DIR}/${profile}/`;
 
     for (const blob of await files.list(prefix)) {
       const rest = blob.key.slice(prefix.length);
@@ -147,9 +50,7 @@ export async function planMoves(
       if (head === undefined) continue;
 
       // The credential store is merged rather than moved, and the old copy is
-      // deliberately left behind. `state.kv` and `audit.log` are per profile and
-      // become the workspace's, but their contents already carry the profile in
-      // every record, so they are concatenated by moving the objects across.
+      // deliberately left behind.
       if (head === 'credentials.enc' || head === 'credentials.enc.key') continue;
 
       // The instance this profile's single-instance surfaces became. Both are
@@ -159,26 +60,41 @@ export async function planMoves(
       // which is the worst of the collisions because the wrong answer is a
       // credential (ADR-059).
       if (head === 'vault.enc') {
-        moves.push({ from: blob.key, to: layout.vault(instanceOf(mapping, 'vault')) });
+        moves.push({ from: blob.key, to: C3.vault(instanceOf(mapping, 'vault')) });
         continue;
       }
       if (head === 'vault.enc.key') {
-        moves.push({ from: blob.key, to: `${layout.vault(instanceOf(mapping, 'vault'))}.key` });
+        moves.push({ from: blob.key, to: `${C3.vault(instanceOf(mapping, 'vault'))}.key` });
         continue;
       }
       if (head === 'skills.d') {
         moves.push({
           from: blob.key,
-          to: `${layout.skills(instanceOf(mapping, 'skills'))}/${tail.join('/')}`,
+          to: `${C3.skills(instanceOf(mapping, 'skills'))}/${tail.join('/')}`,
         });
         continue;
       }
+      // Under ADR-030 a manifest lived in the profile, so an operator using
+      // their own connector in two profiles holds two copies of one file. They
+      // are the same manifest; refusing byte-identical files and asking for the
+      // data directory's layout was not a useful answer.
       if (head === 'providers.d') {
-        moves.push({ from: blob.key, to: `${layout.providers()}/${tail.join('/')}` });
+        const move = claim(claimed, {
+          from: blob.key,
+          to: `${C3.providers()}/${tail.join('/')}`,
+        });
+        if (move !== null) moves.push(move);
+        continue;
+      }
+      // One object per event under a key already carrying the timestamp, so
+      // concatenating three profiles' logs is exactly moving them across.
+      if (head === 'audit.log') {
+        moves.push({ from: blob.key, to: `${C3.audit()}/${tail.join('/')}` });
         continue;
       }
-      if (head === 'state.kv' || head === 'audit.log') {
-        moves.push({ from: blob.key, to: `${DATA_DIR}/${head}/${tail.join('/')}` });
+      if (head === 'state.kv') {
+        const move = stateMove(blob.key, tail, mapping, claimed);
+        if (move !== null) moves.push(move);
         continue;
       }
 
@@ -190,7 +106,7 @@ export async function planMoves(
       // This profile's old key, through this profile's mapping.
       const settled = mapping.get(`${head}.${connection}`);
       const id = settled === undefined ? connection : (settled.split('.')[1] ?? connection);
-      moves.push({ from: blob.key, to: `${DATA_DIR}/${head}/${id}/${tail.slice(1).join('/')}` });
+      moves.push({ from: blob.key, to: `${LEGACY_DATA_DIR}/${head}/${id}/${tail.slice(1).join('/')}` });
     }
   }
 
@@ -198,31 +114,120 @@ export async function planMoves(
   return moves;
 }
 
+
 /**
- * Two objects aimed at one key, caught while this is still a plan.
+ * Where one object under a profile's `state.kv` is going.
+ *
+ * `state.kv` used to be moved verbatim, on the grounds that its records "already
+ * carry the profile". They do not. `connections.v1` is keyed on
+ * `<provider>.<id>` — precisely what the hoist renames — so every profile's
+ * owner layer wrote `connections.v1/vault.main`, and three profiles aimed three
+ * objects at one key. That is the collision `assertOneObjectPerDestination`
+ * called unreachable, and it was reachable from any workspace with two profiles
+ * in it.
  *
- * `applyMoves` checks the destination per object as well, but that check cannot
- * see a collision between two objects *in this run* once the moves are applied
- * concurrently — both would look at an absent destination and both would write.
- * Hoisting it here also puts it where this file says it belongs: everything that
- * can fail happens before the first byte moves, so a refusal leaves the
- * workspace exactly as it was.
+ * Keys are decoded rather than pattern-matched: on disk both halves are
+ * percent-encoded per segment (`connections%2Ev1/vault%2Emain.json`), and
+ * respelling that here would be a second copy of an encoding that must have one.
+ *
+ * `null` means leave it where it is — nothing is deleted by not moving it, and
+ * `doctor` names what is left.
  */
-function assertOneObjectPerDestination(moves: readonly Move[]): void {
-  const seen = new Map<string, string>();
+function stateMove(
+  from: string,
+  tail: readonly string[],
+  mapping: ReadonlyMap<string, string>,
+  claimed: Set<string>,
+): Move | null {
+  const here = `${C3.state()}/${tail.join('/')}`;
+  const leaf = tail[tail.length - 1];
+
+  // Not an object this module wrote: moved as it is, and still held to the
+  // one-object-per-destination rule, because an unrecognised collision is a
+  // thing to refuse rather than to resolve by guessing.
+  if (tail.length < 2 || leaf === undefined || !leaf.endsWith('.json')) {
+    return { from, to: here };
+  }
 
-  for (const move of moves) {
-    const first = seen.get(move.to);
-    if (first !== undefined) {
-      throw new ConfigError(
-        `Two objects want to be at ${move.to}, and this migration cannot merge them.\n` +
-          `  ${first} and ${move.from}. Nothing has been written.\n` +
-          '  This should be unreachable: the hoist gives every profile its own instance of ' +
-          'each owner-layer surface. Please report it with the layout of your data directory.',
-      );
-    }
-    seen.set(move.to, move.from);
+  // `decodeSegment` is `decodeURIComponent`, which throws `URIError` on a stray
+  // `%`. Aborting somebody's whole migration on `URI error`, naming no file, is
+  // not an answer — an undecodable key is one this does not understand, and
+  // those are moved as they are.
+  let segments: string[];
+  let key: string;
+  try {
+    segments = tail.slice(0, -1).map(decodeSegment);
+    key = decodeSegment(leaf.slice(0, -'.json'.length));
+  } catch {
+    return { from, to: here };
+  }
+  const namespace = segments.join('/');
+
+  // Keyed on the provider id, not a connection, so two profiles on the same
+  // vendor hold two entries under one key. `open.ts` treats a miss and a corrupt
+  // entry alike as "not discovered yet" and `connect` refreshes it: first wins.
+  if (namespace === 'discovery') {
+    return claim(claimed, { from, to: here });
   }
+
+  // `<provider>/<connection>` — a provider's own state, keyed on the connection
+  // exactly as its blobs are. Left out of the first pass, so two profiles
+  // holding one provider's fixed-name object (`dav`'s home, `bunq`'s session)
+  // still aimed at one key.
+  if (segments.length === 2) {
+    const [provider, connection] = segments as [string, string];
+    const settled = mapping.get(`${provider}.${connection}`);
+    if (settled === undefined) return { from, to: here };
+
+    const id = settled.slice(settled.indexOf('.') + 1);
+    return claim(claimed, { from, to: `${C3.state()}/${objectKey(`${provider}/${id}`, key)}` });
+  }
+
+  if (namespace !== CONNECTIONS_NAMESPACE) return { from, to: here };
+
+  // This profile's old key, through this profile's mapping — the same lookup
+  // the provider-blob branch does, and wrong in the same way if it is skipped.
+  const settled = mapping.get(key);
+
+  // Orphaned under contract 2 already — no row, and `hoistConnections` reads
+  // rows — so hoisting it would manufacture a connection nothing grants, and its
+  // key can collide with a rename that is real.
+  if (settled === undefined) return null;
+
+  const to = `${C3.state()}/${objectKey(CONNECTIONS_NAMESPACE, settled)}`;
+
+  // Two profiles on one account merge into one row, so both hold a record for
+  // it. One connection; the second differs only in when it was written.
+  if (settled === key) return claim(claimed, { from, to });
+
+  const dot = settled.indexOf('.');
+  const provider = settled.slice(0, dot);
+  const id = settled.slice(dot + 1);
+  return claim(claimed, { from, to, rewrite: (data) => retarget(data, provider, id) });
+}
+
+
+/**
+ * A connection record, told what it is now called.
+ *
+ * Spread rather than assigned field by field, so key order and anything a later
+ * version added survive. Bytes that are not a JSON object are returned
+ * untouched: refusing to move what would not parse strands it under a profile
+ * directory nothing reads.
+ */
+function retarget(data: Uint8Array, provider: string, id: string): Uint8Array {
+  let record: unknown;
+  try {
+    record = JSON.parse(new TextDecoder().decode(data));
+  } catch {
+    return data;
+  }
+
+  if (record === null || typeof record !== 'object' || Array.isArray(record)) return data;
+
+  return new TextEncoder().encode(
+    JSON.stringify({ ...(record as Record<string, unknown>), provider, id }),
+  );
 }
 
 /**
@@ -254,75 +259,4 @@ function instanceOf(mapping: ReadonlyMap<string, string>, provider: string): str
   return 'main';
 }
 
-/**
- * How many objects are in flight at once.
- *
- * Serial was fine while this only ever ran against a local disk. A deployed
- * workspace's audit log is one object per event, so the first real bucket this
- * migrated held 1,906 of them — three round trips each, in series, is minutes of
- * a deploy spent with nothing on screen. The same 16 the read paths in
- * `#providers/memory` and `#providers/tasks` settled on, and for the same
- * reason: enough to hide the latency, not enough to look like an incident to the
- * other end.
- *
- * Safe to widen only while each move stays independent, which is what
- * `assertOneObjectPerDestination` guarantees.
- */
-const MOVE_CONCURRENCY = 16;
-
-export async function applyMoves(files: BlobStore, moves: readonly Move[]): Promise<void> {
-  const pending = moves.filter((move) => move.from !== move.to);
-
-  for (let start = 0; start < pending.length; start += MOVE_CONCURRENCY) {
-    await Promise.all(
-      pending.slice(start, start + MOVE_CONCURRENCY).map((move) => applyMove(files, move)),
-    );
-  }
-}
-
-/** One object, moved or finished. Never deletes before the copy reads back. */
-async function applyMove(files: BlobStore, move: Move): Promise<void> {
-  const data = await files.get(move.from);
-  if (data === null) return;
-
-  if (await files.has(move.to)) {
-    const held = await files.get(move.to);
-
-    // Raced away between the two calls, so there is nothing there after all and
-    // the ordinary path below is still the right one.
-    if (held !== null) {
-      if (!sameBytes(held, data)) {
-        throw new ConfigError(
-          `Two objects want to be at ${move.to}, and this migration cannot merge them.\n` +
-            `  ${move.from} is the second, and what is already there is not a copy of it.\n` +
-            '  Nothing has been deleted. This should be unreachable: the hoist gives every ' +
-            'profile its own instance of each owner-layer surface. Please report it with the ' +
-            'layout of your data directory.',
-        );
-      }
-
-      // Already copied, by a run that did not get to the delete.
-      await files.delete(move.from);
-      return;
-    }
-  }
-
-  await files.put(move.to, data);
-  if ((await files.get(move.to)) === null) {
-    throw new ConfigError(
-      `${move.to} did not read back after being written. Nothing has been deleted; ` +
-        `fix the store and run this again.`,
-    );
-  }
-
-  await files.delete(move.from);
-}
-
-function sameBytes(left: Uint8Array, right: Uint8Array): boolean {
-  if (left.length !== right.length) return false;
-  for (let index = 0; index < left.length; index += 1) {
-    if (left[index] !== right[index]) return false;
-  }
-  return true;
-}
 

package/src/cli/contract3-layout.ts

@@ -0,0 +1,46 @@
+import { LEGACY_DATA_DIR, LEGACY_WORKSPACE_FILE } from '#profile';
+
+/**
+ * Where contract 3 put things, frozen.
+ *
+ * `#profile`'s `layout` describes where things live *now*. A migration lives
+ * between two layouts and must not ask it: `contract3.ts` migrates to this one
+ * and `contract4.ts` migrates away from it, so both need it spelled somewhere
+ * that will not move when the live one does. Asking `layout` made the contract-3
+ * migration write a contract-4 tree, skipping the step contract 4 was about to
+ * take and leaving nothing for it to find.
+ *
+ * One module rather than a copy in each, for the reason `layout.ts` exists: the
+ * credential path was spelled in three files and they have to agree forever.
+ */
+/**
+ * The owner layer's ids through contract 3, before the `lanes_` prefix.
+ *
+ * Spelled here for the reason every other path in this file is: a migration
+ * reads the shape it is migrating *from*. `RESERVED_PROVIDER_IDS` is the live
+ * list and is `lanes_memory` now, so asking it whether a contract-2 row is
+ * owner-layer answers no for every one of them.
+ */
+export const C3_OWNER_PROVIDERS: readonly string[] = [
+  'memory',
+  'tasks',
+  'assets',
+  'skills',
+  'vault',
+  'setup',
+  'identity',
+  'entities',
+];
+
+export const C3 = {
+  /** The registry, before it became `workspaces.yaml`. */
+  workspace: LEGACY_WORKSPACE_FILE,
+  /** A profile's declaration, before it moved inside its own directory. */
+  profile: (profile: string): string => `profiles/${profile}.yaml`,
+  credentials: (): string => `${LEGACY_DATA_DIR}/credentials.enc`,
+  state: (): string => `${LEGACY_DATA_DIR}/state.kv`,
+  audit: (): string => `${LEGACY_DATA_DIR}/audit.log`,
+  providers: (): string => `${LEGACY_DATA_DIR}/providers.d`,
+  vault: (connection: string): string => `${LEGACY_DATA_DIR}/vault.d/${connection}.enc`,
+  skills: (connection: string): string => `${LEGACY_DATA_DIR}/skills.d/${connection}`,
+} as const;

package/src/cli/contract3-shape.ts

@@ -1,6 +1,7 @@
 import { RESERVED_PROVIDER_IDS } from '#connectivity';
 import type { ContractRename, LegacyConnection, LegacyProfile } from './contract3.ts';
 import { keyOf } from './contract3.ts';
+import { C3_OWNER_PROVIDERS } from './contract3-layout.ts';
 
 /**
  * Turning contract 2's shape into contract 3's, without touching a file.
@@ -49,7 +50,7 @@ export function hoistConnections(profiles: ReadonlyMap<string, LegacyProfile>):
       // merge. That is the one outcome ADR-059 forbids: interleaving two sets of
       // notes is not reversible and not reviewable, and for the vault the wrong
       // answer is a credential. Keying on the profile forces a rename instead.
-      const owner = RESERVED_PROVIDER_IDS.includes(connection.provider);
+      const owner = C3_OWNER_PROVIDERS.includes(connection.provider);
       const identity = owner
         ? `${connection.provider} @${profile}`
         : `${connection.provider} ${connection.account}`;
@@ -166,7 +167,20 @@ export function grantsFor(
     return pattern === '*' || pattern.startsWith(`${provider}.`);
   };
 
-  return (config.connections ?? []).map((connection) => {
+  // Keyed on the *settled* connection, because two rows can become one. A
+  // profile could hold `gmail.main` and `gmail.archive` for the same mailbox at
+  // different scopes — which is legal under contract 2, and the reason tokens
+  // are per connection at all — and the hoist keys on provider-plus-account, so
+  // both settle on one row. Emitting a grant per original row then wrote the
+  // same `connection:` twice and `assertGrantsUnique` refused the profile,
+  // failing the migration on a merge it had just performed itself.
+  //
+  // The rules are unioned rather than picked between. Both rows were in force
+  // before, the connection they describe is now one connection, and dropping
+  // either would narrow a policy the operator never narrowed.
+  const rows = new Map<string, { connection: string; allow: Rule[]; deny: Rule[] }>();
+
+  for (const connection of config.connections ?? []) {
     const provider = connection.provider;
     // A bare `*` becomes the provider wildcard rather than being copied
     // through. It would still mean the same thing inside a row, which is
@@ -177,10 +191,22 @@ export function grantsFor(
       return rule.capability === '*' ? { ...rule, capability: `${provider}.*` } : rule;
     };
 
-    return {
-      connection: mapping.get(keyOf(connection)) ?? keyOf(connection),
-      allow: allow.filter((rule) => covers(rule, provider)).map(widen),
-      deny: deny.filter((rule) => covers(rule, provider)).map(widen),
-    };
-  });
+    const key = mapping.get(keyOf(connection)) ?? keyOf(connection);
+    const row = rows.get(key) ?? { connection: key, allow: [], deny: [] };
+
+    row.allow = union(row.allow, allow.filter((rule) => covers(rule, provider)).map(widen));
+    row.deny = union(row.deny, deny.filter((rule) => covers(rule, provider)).map(widen));
+    rows.set(key, row);
+  }
+
+  return [...rows.values()];
+}
+
+/** Rules from two merged rows, keeping the first spelling of each capability. */
+function union(held: readonly Rule[], adding: readonly Rule[]): Rule[] {
+  const merged = [...held];
+  for (const rule of adding) {
+    if (!merged.some((one) => capabilityOf(one) === capabilityOf(rule))) merged.push(rule);
+  }
+  return merged;
 }