@lanes-sh/link 0.7.2 → 0.8.0

package/src/cli/config-templates.ts

@@ -0,0 +1,198 @@
+import { SUPPORTED_CONTRACT } from '#profile';
+
+/**
+ * The files a fresh workspace and a fresh profile are written from.
+ *
+ * Split out of `config-edit.ts` because they are prose rather than machinery:
+ * that file knows how to edit YAML without disturbing what an operator wrote,
+ * and these are the comments an operator reads. Keeping both in one file put it
+ * over the size budget, and the seam was already there.
+ *
+ * **The template and the repair must write a row in one spelling.** Two
+ * spellings of one row is how they drift apart, and `config-edit.test.ts`
+ * asserts that a fresh profile and workspace need no repair, which is the check
+ * that catches it (ADR-050).
+ */
+
+export function newProfileTemplate(profile: string, port: number, subject?: string): string {
+  return `# Lanes Link profile: ${profile}
+#
+# A profile is a *selection*: which of the workspace's accounts this agent may
+# reach, what it may do with each, and who may use it. The accounts themselves
+# live in connections.yaml beside this file, because authorising an account and
+# deciding what may be done with it are two different acts (ADR-057).
+#
+# This file never contains a credential value — only "_ref" pointers into the
+# credential store, which is the workspace's and is encrypted at rest.
+#
+# Edit it by hand or through the CLI; both are supported, and CLI edits preserve
+# your comments and ordering.
+contract: 3
+
+instance:
+  profile: ${profile}
+  port: ${port}
+  host: 127.0.0.1
+
+# What this profile is for, in your own words. Members see it, and so does
+# setup_overview — "reads my mail, keeps the calendar, never sends" is what
+# somebody needs to know before accepting it.
+# description: 
+
+# This file says nothing about where it runs, and that is the point.
+#
+# A profile lives in exactly one workspace, and that workspace declares its own
+# adapters, once, in lanes-link.yaml beside the profiles/ directory (ADR-052).
+# Moving this profile somewhere else is copying the file there.
+#
+#     lanes link status --profile ${profile} --workspace <name>
+#
+# The bearer token below is for CI. People sign in instead: a client that asks
+# for authorization is sent to the Lanes login, and comes back as somebody
+# (ADR-062). "lanes link token show" is for a runner with no browser.
+auth:
+  mode: bearer
+  token_ref: profile/token
+  authorization:
+    mode: self
+
+limits:
+  requests_per_minute: 120      # per profile
+  upstream_calls_per_minute: 60 # per connection, protects vendor quota
+
+# One row per connection this profile may reach, and what it may do with each.
+#
+# A row is the grant. There is no separate list of accounts and list of rules
+# that have to agree — naming a connection here is what makes it reachable, and
+# the allow list is what makes any of its capabilities callable. An account the
+# workspace holds and this file does not name is simply absent: not denied, not
+# advertised, not there.
+#
+# Rules name capabilities of that row's own provider. "gmail.*" covers
+# everything Gmail offers *for that one account*, which is what lets a second
+# row over a second mailbox allow something different (ADR-058).
+#
+# The seven below hold no account, and that is why they are here already: they
+# reach your own material rather than anybody's API, so there was never anything
+# for a connect step to authorise (ADR-050). What each one is:
+#
+#   memory   what you want remembered between sessions
+#   tasks    what you have to do, each with a status
+#   assets   files you want kept, by name
+#   skills   procedures you have written, handed to an agent as instructions
+#   vault    passwords and API keys, released one at a time
+#   setup    what is connected here, and what connecting more would take
+#   entities the people, companies and projects you deal with, and how to
+#            reach each of them — so an agent looks an address up rather
+#            than recalling one
+#
+# To switch one off, add it to that row's deny — deleting the row no longer
+# works, because the next connect or deploy puts it back. The three narrowings
+# worth knowing:
+#
+#   deny: [memory.write]        remember nothing new
+#   deny: [skills.manage.*]     invoke procedures, do not write them
+#   deny: [vault.put, vault.remove]
+grants:
+  - { connection: memory.main, allow: [memory.*], deny: [] }
+  - { connection: tasks.main, allow: [tasks.*], deny: [] }
+  - { connection: assets.main, allow: [assets.*], deny: [] }
+  - { connection: skills.main, allow: [skills.*], deny: [] }
+  - { connection: vault.main, allow: [vault.*], deny: [] }
+  - { connection: setup.main, allow: [setup.*], deny: [] }
+  - { connection: entities.main, allow: [entities.*], deny: [] }
+
+# Who may consume this profile (ADR-060).
+#
+# Empty is nobody, not everybody — default deny on the identity axis. A caller
+# proves who they are by signing in to Lanes, and reaches this profile only if
+# their subject is listed here.
+#
+# "owner" may edit this list. Both roles reach exactly what the grants above
+# allow: a role that changed what an agent could call would be a second policy
+# system beside grants, answering a question the first one already answers.
+members:${
+    subject
+      ? `
+  - { subject: ${subject}, role: owner }`
+      : ' []'
+  }
+`;
+}
+
+export function newWorkspaceTemplate(): string {
+  return `# Lanes Link workspace
+#
+# A workspace holds the accounts you have authorised (connections.yaml) and the
+# profiles that select among them (profiles/). One endpoint serves all of them:
+# every call names the profile it means, with --profile.
+#
+# "workspaces:" below says where this one's bytes go, once, for every profile in
+# it — a profile says nothing about where it runs, so there is one copy of it and
+# nothing to keep in step (ADR-052).
+#
+# A workspace somewhere else is a pointer, and "deploy" writes one:
+#
+#   workspaces:
+#     cloud:
+#       at: gs://your-bucket
+#       lanes_workspace: <id>     # whose members may be delegated to
+#
+# The workspace at that address declares its own adapters, and is the only thing
+# that does. Reading it is a network call, which is why "--workspace cloud" needs
+# that bucket reachable.
+#
+# default_workspace is used when --workspace is absent, and every command that
+# uses it prints which one it got. Commands that publish or destroy — deploy,
+# sync, secrets push, profile remove, disconnect, token rotate — refuse it and
+# make you type the name (ADR-061).
+contract: 3
+default_workspace: local
+workspaces:
+  local:
+    credentials: { adapter: file }
+    storage: { adapter: filesystem }
+`;
+}
+
+/**
+ * `connections.yaml` for a workspace that has just been created.
+ *
+ * The owner layer arrives here rather than in the profile, because these are
+ * connections now (ADR-059) and a second profile should select the same stores
+ * rather than get its own empty ones. `ensureOwnerLayer` keeps this in one
+ * spelling with the repair, which `config-edit.test.ts` asserts by checking that
+ * a fresh workspace needs no repair.
+ */
+export function newConnectionsTemplate(): string {
+  return `# Lanes Link connections
+#
+# Every account authorised in this workspace, in one place. A profile names the
+# ones it may reach in its own "grants:" block — connecting an account and
+# deciding what may be done with it are two acts, and only the second belongs to
+# a profile (ADR-057).
+#
+# "account" is the identity the provider reports — an address, a workspace — so
+# this list says whose data is reachable without having to look anything up.
+# "label" is your own word for the same row, and only ever displayed.
+#
+# The seven below hold no account: they reach your own material rather than
+# anybody's API, so there was never anything for a connect step to authorise
+# (ADR-050). Make a second one — "lanes link connect memory --id work" — when you
+# want two profiles to share nothing.
+contract: 3
+
+connections:
+  - { id: main, provider: memory, account: Memory }
+  - { id: main, provider: tasks, account: Tasks }
+  - { id: main, provider: assets, account: Assets }
+  - { id: main, provider: skills, account: Skills }
+  - { id: main, provider: vault, account: Vault }
+  - { id: main, provider: setup, account: Setup }
+  - { id: main, provider: entities, account: Entities }
+
+# App registrations, shared by every connection of that vendor.
+oauth_apps: {}
+`;
+}
+

package/src/cli/contract3-data.ts

@@ -0,0 +1,328 @@
+import { ConfigError, DATA_DIR, isRemoteWorkspace, layout } from '#profile';
+import { createFileSecretStore } from '#secrets';
+import type { BlobStore } from '#stores/blobs';
+
+/**
+ * The half of the contract-3 migration that moves bytes rather than YAML.
+ *
+ * Split from `contract3.ts` on the seam the migration already has: that file
+ * decides *what* the new shape is, and this one carries the credentials and
+ * objects into it. Both halves are ordered so a crash between any two steps
+ * leaves a workspace that still opens, and the rule that makes that true lives
+ * here — nothing is deleted until what replaced it has been read back.
+ */
+
+export interface Move {
+  readonly from: string;
+  readonly to: string;
+}
+
+/** `gmail.main` — how a connection is addressed in every file after this. */
+function keyOf(connection: { provider: string; id: string }): string {
+  return `${connection.provider}.${connection.id}`;
+}
+
+/**
+ * Which credential refs the merged store will hold.
+ *
+ * Read-only, and it runs before anything is written so the report an operator
+ * confirms is the real one. A ref present in two profiles' stores with two
+ * different values is the one thing this cannot resolve, and it is reported
+ * rather than merged: both are real credentials, and picking either would point
+ * a connection at the wrong account's token.
+ *
+ * **A workspace in a bucket has nothing to merge, and this says so rather than
+ * finding out.** `workspacePath` refuses a filesystem adapter against a remote
+ * root, so the only credential store such a workspace can declare is
+ * `gcp-secret-manager` — whose refs were never scoped by profile, and are
+ * therefore already what contract 3 wants. Without the guard the path below is
+ * built by string interpolation into `gs://bucket/data/<profile>/credentials.enc`
+ * and handed to `Bun.file`, where the failure is swallowed by the `catch` and
+ * reads exactly like a workspace with no credentials in it.
+ */
+export async function planCredentials(root: string, profiles: readonly string[]): Promise<string[]> {
+  if (isRemoteWorkspace(root)) return [];
+
+  const refs = new Set<string>();
+
+  for (const profile of profiles) {
+    const store = createFileSecretStore({ path: `${root}/${DATA_DIR}/${profile}/credentials.enc` });
+    try {
+      for (const ref of await store.list()) refs.add(ref);
+    } catch {
+      // A store that will not open is reported by `doctor`, not here. This is a
+      // preview and must not fail on a workspace that is already broken.
+    }
+  }
+
+  return [...refs].sort();
+}
+
+/**
+ * Copy every profile's credentials into the workspace store.
+ *
+ * Written and read back before the old stores are touched, which is the whole
+ * of the safety argument: a half-finished merge that has not deleted anything is
+ * recoverable by running it again, and one that deleted first is not.
+ *
+ * The old stores are left in place regardless. They are a few kilobytes, they
+ * are the only copy of anything if this went wrong, and `doctor` names them so
+ * an operator can remove them once the endpoint has served a request.
+ *
+ * Skipped entirely for a workspace in a bucket, for the reason `planCredentials`
+ * gives: its credentials are in Secret Manager under refs that were never
+ * per-profile, so there is no second store to fold in.
+ */
+export async function mergeCredentials(root: string, profiles: readonly string[]): Promise<void> {
+  if (isRemoteWorkspace(root)) return;
+
+  const destination = createFileSecretStore({ path: `${root}/${layout.credentials()}` });
+
+  for (const profile of profiles) {
+    const source = createFileSecretStore({ path: `${root}/${DATA_DIR}/${profile}/credentials.enc` });
+
+    let refs: string[];
+    try {
+      refs = await source.list();
+    } catch {
+      continue;
+    }
+
+    for (const ref of refs) {
+      const value = await source.get(ref);
+      if (value === null) continue;
+
+      const held = await destination.get(ref);
+      if (held !== null) {
+        if (held === value) continue;
+        throw new ConfigError(
+          `Two profiles hold different values for the credential "${ref}", and this migration ` +
+            `cannot choose between them.\n` +
+            `  Both are real credentials for different accounts, and picking either would point a ` +
+            `connection at the wrong one.\n` +
+            `  Rename one connection before migrating, so its credential ref differs.`,
+        );
+      }
+
+      await destination.set(ref, value);
+      if ((await destination.get(ref)) !== value) {
+        throw new ConfigError(
+          `The credential "${ref}" did not read back after being written to ` +
+            `${layout.credentials()}. Nothing has been deleted; fix the store and run this again.`,
+        );
+      }
+    }
+  }
+}
+
+/**
+ * Where every object under `data/<profile>/` is going.
+ *
+ * Driven by `perProfile`, the per-profile map the hoist already built from old
+ * key to new — because the rename that matters is *this profile's*. The first
+ * version of this keyed a lookup by the hoisted (new) key and queried it with
+ * the old one, which made the resolution an unconditional no-op: provider and
+ * connection ids contain no dot, so anything the map returned already had the id
+ * being looked up. Two profiles holding `gmail.main` for different mailboxes
+ * both sent their blobs to `data/gmail/main/`, and the second one's landed in
+ * the first one's namespace.
+ *
+ * Anything that matches no rule is left exactly where it is: this moves what it
+ * understands and never deletes what it does not.
+ */
+export async function planMoves(
+  files: BlobStore,
+  profiles: readonly string[],
+  perProfile: ReadonlyMap<string, ReadonlyMap<string, string>>,
+): Promise<Move[]> {
+  const moves: Move[] = [];
+
+  for (const profile of profiles) {
+    const mapping = perProfile.get(profile) ?? new Map<string, string>();
+    const prefix = `${DATA_DIR}/${profile}/`;
+
+    for (const blob of await files.list(prefix)) {
+      const rest = blob.key.slice(prefix.length);
+      const [head, ...tail] = rest.split('/');
+      if (head === undefined) continue;
+
+      // The credential store is merged rather than moved, and the old copy is
+      // deliberately left behind. `state.kv` and `audit.log` are per profile and
+      // become the workspace's, but their contents already carry the profile in
+      // every record, so they are concatenated by moving the objects across.
+      if (head === 'credentials.enc' || head === 'credentials.enc.key') continue;
+
+      // The instance this profile's single-instance surfaces became. Both are
+      // one store per profile in contract 2 and one per *connection* in
+      // contract 3, so two profiles' vaults are two documents — sending both to
+      // `vault('main')` orphaned the second and silently gave it the first's,
+      // which is the worst of the collisions because the wrong answer is a
+      // credential (ADR-059).
+      if (head === 'vault.enc') {
+        moves.push({ from: blob.key, to: layout.vault(instanceOf(mapping, 'vault')) });
+        continue;
+      }
+      if (head === 'vault.enc.key') {
+        moves.push({ from: blob.key, to: `${layout.vault(instanceOf(mapping, 'vault'))}.key` });
+        continue;
+      }
+      if (head === 'skills.d') {
+        moves.push({
+          from: blob.key,
+          to: `${layout.skills(instanceOf(mapping, 'skills'))}/${tail.join('/')}`,
+        });
+        continue;
+      }
+      if (head === 'providers.d') {
+        moves.push({ from: blob.key, to: `${layout.providers()}/${tail.join('/')}` });
+        continue;
+      }
+      if (head === 'state.kv' || head === 'audit.log') {
+        moves.push({ from: blob.key, to: `${DATA_DIR}/${head}/${tail.join('/')}` });
+        continue;
+      }
+
+      // Otherwise it is `<provider>/<connection>/...`, the namespace every
+      // provider's blobs are scoped into.
+      const connection = tail[0];
+      if (connection === undefined) continue;
+
+      // This profile's old key, through this profile's mapping.
+      const settled = mapping.get(`${head}.${connection}`);
+      const id = settled === undefined ? connection : (settled.split('.')[1] ?? connection);
+      moves.push({ from: blob.key, to: `${DATA_DIR}/${head}/${id}/${tail.slice(1).join('/')}` });
+    }
+  }
+
+  assertOneObjectPerDestination(moves);
+  return moves;
+}
+
+/**
+ * Two objects aimed at one key, caught while this is still a plan.
+ *
+ * `applyMoves` checks the destination per object as well, but that check cannot
+ * see a collision between two objects *in this run* once the moves are applied
+ * concurrently — both would look at an absent destination and both would write.
+ * Hoisting it here also puts it where this file says it belongs: everything that
+ * can fail happens before the first byte moves, so a refusal leaves the
+ * workspace exactly as it was.
+ */
+function assertOneObjectPerDestination(moves: readonly Move[]): void {
+  const seen = new Map<string, string>();
+
+  for (const move of moves) {
+    const first = seen.get(move.to);
+    if (first !== undefined) {
+      throw new ConfigError(
+        `Two objects want to be at ${move.to}, and this migration cannot merge them.\n` +
+          `  ${first} and ${move.from}. Nothing has been written.\n` +
+          '  This should be unreachable: the hoist gives every profile its own instance of ' +
+          'each owner-layer surface. Please report it with the layout of your data directory.',
+      );
+    }
+    seen.set(move.to, move.from);
+  }
+}
+
+/**
+ * Copy, verify, then delete. In that order, per object.
+ *
+ * A move that deleted first would lose an object on any failure, and these are
+ * the owner's notes, tasks and entities.
+ *
+ * A destination that already holds *different* bytes is a bug rather than a case
+ * to handle, now that the hoist gives every profile's owner layer its own
+ * instance: two sets of notes can no longer be aimed at one key. It used to be
+ * skipped silently, which is how work's vault came to be orphaned while `moved`
+ * reported it as moved. `assertOneObjectPerDestination` refuses that while this
+ * is still a plan, and the check here catches what a plan cannot see.
+ *
+ * **The same bytes at the destination is the interrupted move, and it finishes
+ * it.** Copy-then-delete has a window between the two, and this migration now
+ * runs against buckets — where the window is a network round trip rather than a
+ * syscall, and an interruption is something that happens rather than something
+ * to reason about. Refusing there would have meant a workspace that could not be
+ * migrated by running the migration again, which is the one recovery this file
+ * promises.
+ */
+/** Which instance of a single-instance surface this profile's store became. */
+function instanceOf(mapping: ReadonlyMap<string, string>, provider: string): string {
+  for (const [from, to] of mapping) {
+    if (from.startsWith(`${provider}.`)) return to.split('.')[1] ?? 'main';
+  }
+  return 'main';
+}
+
+/**
+ * How many objects are in flight at once.
+ *
+ * Serial was fine while this only ever ran against a local disk. A deployed
+ * workspace's audit log is one object per event, so the first real bucket this
+ * migrated held 1,906 of them — three round trips each, in series, is minutes of
+ * a deploy spent with nothing on screen. The same 16 the read paths in
+ * `#providers/memory` and `#providers/tasks` settled on, and for the same
+ * reason: enough to hide the latency, not enough to look like an incident to the
+ * other end.
+ *
+ * Safe to widen only while each move stays independent, which is what
+ * `assertOneObjectPerDestination` guarantees.
+ */
+const MOVE_CONCURRENCY = 16;
+
+export async function applyMoves(files: BlobStore, moves: readonly Move[]): Promise<void> {
+  const pending = moves.filter((move) => move.from !== move.to);
+
+  for (let start = 0; start < pending.length; start += MOVE_CONCURRENCY) {
+    await Promise.all(
+      pending.slice(start, start + MOVE_CONCURRENCY).map((move) => applyMove(files, move)),
+    );
+  }
+}
+
+/** One object, moved or finished. Never deletes before the copy reads back. */
+async function applyMove(files: BlobStore, move: Move): Promise<void> {
+  const data = await files.get(move.from);
+  if (data === null) return;
+
+  if (await files.has(move.to)) {
+    const held = await files.get(move.to);
+
+    // Raced away between the two calls, so there is nothing there after all and
+    // the ordinary path below is still the right one.
+    if (held !== null) {
+      if (!sameBytes(held, data)) {
+        throw new ConfigError(
+          `Two objects want to be at ${move.to}, and this migration cannot merge them.\n` +
+            `  ${move.from} is the second, and what is already there is not a copy of it.\n` +
+            '  Nothing has been deleted. This should be unreachable: the hoist gives every ' +
+            'profile its own instance of each owner-layer surface. Please report it with the ' +
+            'layout of your data directory.',
+        );
+      }
+
+      // Already copied, by a run that did not get to the delete.
+      await files.delete(move.from);
+      return;
+    }
+  }
+
+  await files.put(move.to, data);
+  if ((await files.get(move.to)) === null) {
+    throw new ConfigError(
+      `${move.to} did not read back after being written. Nothing has been deleted; ` +
+        `fix the store and run this again.`,
+    );
+  }
+
+  await files.delete(move.from);
+}
+
+function sameBytes(left: Uint8Array, right: Uint8Array): boolean {
+  if (left.length !== right.length) return false;
+  for (let index = 0; index < left.length; index += 1) {
+    if (left[index] !== right[index]) return false;
+  }
+  return true;
+}
+