@jr2/orchestrator 0.4.1 → 0.6.0

package/docs/concepts/pools-and-sources.md

@@ -0,0 +1,230 @@
+---
+title: Pools and Sources
+description: How a Pool runs one worker Machine for each work item that a Source gives, and how its run ends.
+---
+
+# Pools and Sources
+
+A [Pool](../reference/glossary.md#pool) is a [Machine](../reference/glossary.md#machine) that runs one worker Machine
+for each item of work. It gets the items from a [Source](../reference/glossary.md#source), one at a time, and it runs a
+limited number of workers at once. You make a Pool with [`pool()`](../reference/authoring-surface/pool.md) and a Source
+with [`source()`](../reference/authoring-surface/source.md). A Pool is usually the root of a
+[Workflow](../reference/glossary.md#workflow). For example, a Pool can run one [`task`](../reference/machines.md#task)
+for each ticket that is ready.
+
+## Why it exists
+
+A Machine that works through a list of items does the same work each time. It claims an item and starts a worker for it.
+It runs no more workers than a limit. It collects each result, looks for more work, and decides when the run is done. A
+Pool does all of this. The Machine gives only the Source, the worker and the limit.
+
+The Pool holds both the Source and the workers. Thus it can tell three conditions apart. All the work is done, the work
+that remains can never start, or the workers wait for a person.
+
+## How a Pool behaves
+
+The example is a Pool over a Source of four tickets. T-3 needs T-1 and T-2, and T-4 needs a ticket that does not exist.
+Each worker waits at a [Gate](../reference/glossary.md#gate) until a person approves or rejects its ticket. Thus the
+example runs with no model. A worker is usually a Machine with an [Agent](../reference/glossary.md#agent), such as
+`task`.
+
+```ts
+import { assign, fromPromise } from "xstate";
+import { z } from "zod";
+import { defineEvent, jr2Setup, pool, source, type PoolNextInput } from "@jr2/orchestrator";
+
+type Ticket = { id: string; title: string; needs: string[] };
+
+// A small backlog with dependencies. T-3 can start only when T-1 and T-2 are done.
+// T-4 needs a ticket that does not exist, so it never becomes ready.
+const backlog: Ticket[] = [
+  { id: "T-1", title: "Trim the name.", needs: [] },
+  { id: "T-2", title: "Add a --version flag.", needs: [] },
+  { id: "T-3", title: "Document the two changes.", needs: ["T-1", "T-2"] },
+  { id: "T-4", title: "Remove the old flag.", needs: ["T-9"] },
+];
+
+// The ids that the Pool claimed. This set is in memory, so it serves one run.
+// A real Work Source asks the issue tracker instead.
+const claimed = new Set<string>();
+
+// The Source: give the next ready ticket, or say how many tickets are not ready.
+const tickets = source<Ticket>({
+  next: fromPromise(async ({ input }: { input: PoolNextInput }) => {
+    const done = (id: string) => claimed.has(id) && !input.active.includes(id);
+    const waiting = backlog.filter((t) => !claimed.has(t.id));
+    const ready = waiting.find((t) => t.needs.every(done));
+    if (!ready) return { item: null, open: waiting.length };
+    claimed.add(ready.id);
+    return { item: ready, open: waiting.length - 1 };
+  }),
+  // Send this event to the Pool's `source` Gate when the backlog changes.
+  wake: defineEvent({
+    name: "tickets_changed",
+    description: "The backlog changed. Look for ready tickets again.",
+    audience: "external",
+    input: z.object({}),
+  }),
+});
+
+const approve = defineEvent({
+  name: "approve",
+  description: "Accept the ticket.",
+  audience: "external",
+  input: z.object({}),
+});
+
+const reject = defineEvent({
+  name: "reject",
+  description: "Refuse the ticket.",
+  audience: "external",
+  input: z.object({ reason: z.string().describe("Why, in one sentence.") }),
+});
+
+type Outcome = { id: string; outcome: "approved" } | { id: string; outcome: "rejected"; reason: string };
+
+// The worker: one Machine for each ticket. Here a person decides at a Gate.
+// A worker is usually a Machine with an Agent, such as `task`.
+const review = jr2Setup({
+  types: {} as { input: Ticket; context: Ticket & { outcome?: Outcome }; output: Outcome },
+  events: [approve, reject],
+}).createMachine({
+  id: "review",
+  context: ({ input }) => input,
+  initial: "review",
+  states: {
+    review: {
+      invoke: { src: "gate", input: ({ context }) => ({ meta: { title: context.title } }) },
+      on: {
+        approve: {
+          target: "decided",
+          actions: assign({ outcome: ({ context }) => ({ id: context.id, outcome: "approved" as const }) }),
+        },
+        reject: {
+          target: "decided",
+          actions: assign({
+            outcome: ({ context, event }) => ({ id: context.id, outcome: "rejected" as const, reason: event.reason }),
+          }),
+        },
+      },
+    },
+    decided: { type: "final" },
+  },
+  output: ({ context }) => context.outcome!,
+});
+
+// The Pool: one `review` for each ready ticket, two at most at once.
+export const machine = pool(review, {
+  id: "tickets",
+  source: tickets,
+  itemId: (ticket: Ticket) => ticket.id,
+  cap: 2,
+});
+```
+
+The Pool asks for work, starts workers, and waits:
+
+- The Pool calls the `next` of its Source when the run starts. It gives `next` the ids of the items that its workers
+  have now.
+- When `next` gives an item, the Pool starts a worker for it, and calls `next` again. The id of the worker is
+  `itemId(item)`.
+- When `cap` workers run, the Pool does not call `next`. It calls `next` again when a worker finishes.
+- When no item is ready and workers run, the Pool waits. It calls `next` again when a worker finishes, when the `wake`
+  event comes, or after `pollEvery` milliseconds.
+- When no worker runs and `next` gives no item, the run ends.
+
+In the example, T-1 and T-2 start at once, because `cap` is 2. The run then has three Gates: `T-1.review`, `T-2.review`
+and `source`. When a person answers both tickets, T-3 starts. When T-3 finishes, only T-4 remains, and it can never
+start. Thus the run ends in the state `deadlocked`, and the `status` of its output is `deadlocked`.
+
+### How the run ends
+
+The output of a Pool is `{ status, items, failures }`. `items` holds the output of each worker that got to a final
+state, by item id. `failures` holds the [Failure](../reference/glossary.md#failure) of each worker that failed, by item
+id. The Pool catches a worker's Failure: that worker stops, its Sandbox is deleted, and the other workers go on. A
+defect in a worker is not caught, and ends the run `error`.
+
+When the Pool ends and one or more workers failed, the run ends `failed` by default. This is true for `drained` and for
+`deadlocked`. Its output is then the Failure of the Pool: the reason is `"<n> of <m> items failed"`, and the detail is
+`{ status, items, failures }`. `jr2 run` exits 1. To end `done` with the failures in the output, give `pool()` the
+option `failures: "report"`. Use it when a failed item is a correct answer of the run, for example in a run that tries
+each item and reports which ones did not succeed.
+
+`status` comes from the last answer of `next`:
+
+| The last answer of `next`                       | `status`     | What it means                            |
+| ----------------------------------------------- | ------------ | ---------------------------------------- |
+| `null`, or `{ item: null, open: 0 }`            | `drained`    | No item is left.                         |
+| `{ item: null, open: <n> }`, with `<n>` above 0 | `deadlocked` | Items exist, but none of them can start. |
+
+While workers wait at Gates, the run does not end. It stays open, and the Gates of the run show what needs you. This is
+not a status in the output: a run that waits has no output yet.
+
+### Inputs, Gates and the wake event
+
+- A worker gets its item as its input, or what `itemInput(item, { input })` makes of it. The Pool does not give the run
+  input to a worker. `itemInput` can copy a part of it into the worker's input.
+- The run input of a Pool is its own `input` schema, not the input of its worker. `cap` and `itemInput` read it.
+- The Gates of a worker show in the run. The id of each one starts with the id of its item, such as `T-1.review`.
+- The Pool takes the `wake` event of the Source at a Gate named `source`, for the whole run. A webhook or
+  [`jr2 send`](../reference/cli.md#send) sends it. When the Pool is inside a different Machine, the Gate is
+  `<path>.source`.
+- The [Vocabulary](../reference/glossary.md#vocabulary) of a Pool is only its `wake` event. Each worker keeps its own.
+
+A Pool is a Machine. It can be the root of a Workflow, or a different Machine can compose it.
+[`customize()`](../reference/authoring-surface/customize.md) goes through a Pool to its worker, as
+[Composition](composition.md) tells.
+
+## Sources
+
+[`source(spec)`](../reference/authoring-surface/source.md) declares where a Pool gets its items. A Source has three
+parts ([`SourceSpec`](../reference/authoring-surface/type-sourcespec.md)):
+
+- `next`: an [xstate](https://xstate.js.org) promise actor that claims the next ready item. Its input is
+  `{ active, settled, input }`: the ids of the items that workers have now, the ids of the finished and failed items,
+  and the input of the Pool. Thus a Source can fan out over a plan with no state of its own.
+- `wake`: an optional event, made with [`defineEvent`](../reference/authoring-surface/defineevent.md), that tells the
+  Pool that new work is ready.
+- `pollEvery`: an optional time in milliseconds. While the Pool waits, it calls `next` again at this interval.
+
+`next` resolves to one of three answers ([`SourceNextResult`](../reference/authoring-surface/type-sourcenextresult.md)):
+an item, `null` when no item is ready now, or `{ item, open }`. `open` is the number of items that exist but are not
+ready. Without `open`, the Pool cannot tell a finished run from a blocked one. Thus, when no worker runs, `null` means
+`drained`.
+
+A Source can be a queue, a generator, or a set that `next` queries again each time.
+
+## Work Source and the Ready-set
+
+A [Work Source](../reference/glossary.md#work-source) is a Source over an issue tracker, such as GitHub issues or Jira.
+The Work Source reads the dependencies between the items from the issue tracker. The
+[Ready-set](../reference/glossary.md#ready-set) is the set of items that nothing blocks now. The `next` of a Work Source
+gives the next item of the Ready-set.
+
+The Machine does not hold the dependencies, and it does not copy the Ready-set into its context. It asks the Work Source
+again each time. The Ready-set changes during a run. A person can add a ticket, or open a closed ticket again. Use
+`wake` or `pollEvery`, so that the Pool sees the change while it waits.
+
+jr2 does not include a Work Source. You write its `next` for your issue tracker. In the example, the list in memory is
+the issue tracker.
+
+## Rules
+
+- `itemId` gives each item an id of its own, and the same id each time. The Pool uses it as the id of the worker and as
+  the key in `items`.
+- `next` must not give an item in `active` or in `settled`.
+- `cap` is a number, or a function of the run input. Without `cap`, one worker runs at a time.
+- `onDrained` takes only `"final"`: the run ends.
+- The Pool calls `next` again on `wake` or after `pollEvery` only while it waits: workers run and no item is ready.
+- A `task` in a Pool needs a `branch` in its input. Make it in `itemInput`, for example from the item id.
+
+## Read next
+
+- [Run one worker per item](../guides/run-one-worker-per-item.md)
+- [Composition](composition.md)
+- [Gates](gates.md)
+- Reference: [`pool()`](../reference/authoring-surface/pool.md), [`source()`](../reference/authoring-surface/source.md),
+  [`PoolSpec`](../reference/authoring-surface/type-poolspec.md),
+  [`SourceSpec`](../reference/authoring-surface/type-sourcespec.md),
+  [`SourceNextResult`](../reference/authoring-surface/type-sourcenextresult.md)
+- [ADR-0017](../reference/decisions/0017-pool-over-a-source-port.md): the run loop is a Pool over a Source

package/docs/concepts/secrets-and-trust.md

@@ -0,0 +1,306 @@
+---
+title: Secrets and trust
+description: Where jr2 keeps each credential, which tokens it makes, and what a caller can see and do without one.
+---
+
+# Secrets and trust
+
+jr2 keeps each credential of the `secrets` list, the key of `harness.provider.apiKey`, and each token of its own, out of
+the containers in which an [Agent](../reference/glossary.md#agent) executes code. A
+[Held secret](../reference/glossary.md#held-secret) is a credential that the [Harness](../reference/glossary.md#harness)
+uses through its pod's [Custodian](../reference/glossary.md#custodian), and the Harness container holds only its
+[Stand-in](../reference/glossary.md#stand-in). An [Effect secret](../reference/glossary.md#effect-secret) is a
+credential that an [Effect](../reference/glossary.md#effect) uses, and only an
+[Effector](../reference/glossary.md#effector) holds its value. jr2 also makes its own tokens, for each caller of the
+[Orchestrator](../reference/glossary.md#orchestrator) and of each Harness. Without a token, a caller can see each
+[Machine](../reference/glossary.md#machine) and where each live run is in it, but it cannot control a run.
+
+## Why it exists
+
+An Agent's [Working tools](../reference/glossary.md#working-tools) give it a shell in the Harness container. Thus the
+Agent can run each program of the image, and it can send what it reads out of the pod. If a model API key were in its
+reach, the Agent could take it away. If the credential of the Orchestrator's API were in its reach, the Agent could
+answer its own review [Gate](../reference/glossary.md#gate), or send a pick for a different Agent.
+
+jr2 does not try to hide the API from the Agent. Instead, it keeps the credentials out of the Agent's reach. The
+Instance token enters no Harness pod. Each credential of the `secrets` list, and the key of `harness.provider.apiKey`,
+goes to a container in which the Agent executes nothing. [Architecture](architecture.md) tells where those containers
+run. The Harness itself holds one secret value, and [Confined programs](#confined-programs) tells how jr2 keeps it from
+the Agent's code.
+
+## One list, and the name decides
+
+[`jr2.config.ts`](../reference/config.md) has one [`secrets`](../reference/config.md#secrets) list. An entry goes to
+each place that names it, and to no other place:
+
+```ts
+import { defineConfig } from "@jr2/orchestrator";
+
+export default defineConfig({
+  secrets: [
+    // A model key. Each pod whose Agents use Anthropic claims it, and the Custodian of that pod holds it.
+    // It has no hosts, so it goes only to the host that Anthropic's calls go to.
+    { name: "ANTHROPIC_API_KEY", value: process.env.ANTHROPIC_API_KEY },
+    // A Held secret for a tool that the Agent runs. A workspace() claims it with secrets: ["TRACKER_TOKEN"].
+    // The tool sends it as a bearer. The Custodian puts the value only into requests to this host.
+    { name: "TRACKER_TOKEN", value: process.env.TRACKER_TOKEN, hosts: ["api.tracker.example.com"] },
+    // An Effect secret. An Effect names it with secret("DEPLOY_TOKEN"), and only its Effector holds the value.
+    { name: "DEPLOY_TOKEN", value: process.env.DEPLOY_TOKEN },
+  ],
+});
+```
+
+- A pod claims a Held secret when its [`workspace()`](../reference/authoring-surface/workspace.md) lists the name under
+  `secrets`. A pod also claims the key of each provider that an Agent in that pod uses.
+- The [Instance Harness](../reference/glossary.md#instance-harness) claims only the model keys of the Menu-only Agents
+  that it hosts.
+- An Effect claims an Effect secret when it names it with [`secret()`](../reference/authoring-surface/secret.md).
+- One entry can be a Held secret and an Effect secret at the same time. The placement is a fact of each place that names
+  it, not of the entry.
+- An entry that nothing names goes nowhere, and [`jr2 up`](../reference/cli.md#up) warns about it.
+
+[`harness.provider.apiKey`](../reference/config.md#harness-provider-apikey) is a Held secret too. Each pod whose Agents
+use the custom provider claims it, and it goes only to the host of `harness.provider.baseUrl`.
+
+`jr2 up` reads the values on your computer, from your shell or `.env`. It writes each value only into the Secrets of the
+containers that hold it. An entry can also name a key of a Secret that you manage, with `valueFrom`.
+
+## Held secrets
+
+The Harness of a pod holds one Stand-in for each Held secret of the pod, in a file that only the Harness reads. jr2
+makes a new Stand-in for each pod, and it is secret. It has a meaning only to the Custodian of that pod. A program of an
+Agent gets the Stand-in of a Held secret only when the `workspace()` lists that secret under `secrets`. The fetch
+Stand-in is the one public Stand-in (see below).
+
+When a pod holds a Held secret other than the tokens of its Sandbox, the Harness container sends its HTTPS traffic
+through the Custodian (`HTTPS_PROXY`). For each request to a host that a Held secret is bound to, the Custodian does
+these things:
+
+1. It ends the TLS connection with a certificate that `jr2 up` issued from the CA of the
+   [Instance](../reference/glossary.md#instance).
+2. It replaces the Stand-in with the value, in the headers that the entry names.
+3. It opens its own TLS connection to the host, and verifies the certificate of the host.
+
+The Custodian sends the traffic to each other host with no change. It refuses a loopback, link-local or unspecified
+address. A request to a bound host that has no Stand-in in its headers does not leave the pod.
+
+These fields of an entry bind a Held secret. They do not place it:
+
+- [`hosts`](../reference/config.md#secrets-hosts): the hosts that the value can go to. A known model key with one
+  default host, such as `ANTHROPIC_API_KEY`, can leave it out. It then goes only to the host of its provider.
+- [`headers`](../reference/config.md#secrets-headers): the headers in which the Custodian replaces the Stand-in.
+- [`paths`](../reference/config.md#secrets-paths): the path prefixes that the value can go to.
+
+The Custodian also holds the two tokens of its [Sandbox](../reference/glossary.md#sandbox): the control token and the
+fetch token.
+
+- The Harness reads the [Menu](../reference/glossary.md#menu) and sends each pick with the control Stand-in, and the
+  Custodian replaces it with the control token. The control Stand-in is secret. Only the Harness holds it, in a file
+  that no program of an Agent can read. With each request, the Harness also tells which Agent the conversation is for,
+  and the Orchestrator checks it.
+- A `git fetch` in the Sandbox sends the fetch Stand-in, which is public. The Custodian replaces it with the fetch token
+  only for a fetch. It refuses the fetch Stand-in for the Menu and for a pick.
+
+The Instance Harness has a token of its own for the Menu. It runs no program of an Agent.
+
+### What a Held secret does not stop
+
+- **The Agent's code can spend a key that the `workspace()` lists.** While the pod lives, a program of the Agent can
+  send requests through the Custodian to the bound host of that key. A key that can make new keys gives the Agent a key
+  that it can take away. Use a key with no admin rights. Set narrow `paths`.
+- **A response passes with no change.** If a host puts the full key in a response, the Agent reads it.
+- **jr2 controls egress only at the cloud metadata addresses.** A client that does not use the Custodian sends only the
+  Stand-in, which is worth nothing outside the pod. The cluster decides which hosts a pod can reach. One exception: a
+  NetworkPolicy of jr2 stops each Harness pod at the metadata addresses of the large cloud providers (`169.254.169.254`,
+  `fd00:ec2::254`, `169.254.170.2`, `169.254.170.23` and `100.100.100.200`). There, the metadata service of a node can
+  give the node's cloud credential to a pod. The list is not complete for every provider, and only a CNI that enforces
+  NetworkPolicy applies it. See
+  [Move from kind to a shared cluster](../guides/move-from-kind-to-a-shared-cluster.md#check-the-cluster).
+- **A key that signs the request cannot be a Held secret.** A SigV4 or ADC credential signs the request, and the
+  Custodian cannot put a value into a signature. Use an Effect secret for such a credential. For a model provider, use a
+  gateway that holds the credential, and make the key of the gateway a Held secret.
+- **A tool must trust the CA of the Instance.** When a pod holds a Held secret other than the tokens of its Sandbox, jr2
+  sets the trust variables of Node.js, OpenSSL, Python and git in its Harness container. A client that does not trust
+  the CA, such as a Java client, cannot connect to a bound host. It sends nothing.
+
+## Confined programs
+
+The Harness runs the [Turns](../reference/glossary.md#turn) of each Agent of the Sandbox. The Harness process itself
+runs only jr2's code. Each other process in the Harness container runs confined with Landlock: each command of the
+Agent's shell, and so the tests, the builds, git and its hooks; the search tools; the I/O of the read, write and edit
+tools; jr2's own git; and each `command` Effect that names no Effect secret. Code that one Agent writes in `/work` can
+run in a program of a different Agent, for example when a reviewer runs the tests of a coder. Thus jr2 confines each
+such program, and not only the programs of one Agent.
+
+All these programs are in one Landlock domain. They can see, signal and trace each other, as the programs of one user on
+a computer can. Thus a later shell command can find and stop a server that an earlier command started. When the process
+that starts these programs stops, the Harness starts it again, in a new domain. The programs of the old domain continue,
+and a program of the new domain cannot trace them.
+
+A confined program can do all that the user of the container can do, with these exceptions:
+
+- It cannot read, write, make, remove, move or link a file in `/run/jr2`. That folder holds the control Stand-in, the
+  Stand-in of each Held secret, the socket of the Custodian for the picks, and the conversations of the Harness. A
+  program can list the names in `/run/jr2`, as it can list each folder: `ls /` and `find /` work. The names are not
+  secrets.
+- It cannot make a socket in `/run/jr2`. It can connect to the socket of the Custodian, but the Custodian refuses each
+  request on the socket that does not have the control Stand-in, and the program cannot read that Stand-in.
+- It cannot trace the Harness process, or read its environment or its memory.
+- When the kernel has Landlock ABI 6 or later, it cannot send a signal to the Harness process, or connect to an abstract
+  Unix socket that the Harness or the Custodian made.
+
+Thus a program can write in a folder of the image that its user owns, move a file from one folder to another, and run
+the tools of the image. The kernel keeps the shared parts safe: the caches of each [Repo](../reference/glossary.md#repo)
+at `/repos`, the runtime of jr2 at `/opt/jr2` and the CA bundles are read-only. The pod has no ServiceAccount token. A
+program can connect to each address that the pod can reach.
+
+The environment of a confined program is the environment with which the container started: the `ENV` of the image,
+`harness.env`, `harness.envFrom`, and the proxy and trust variables. jr2 removes its own `JR2_` names from it, except
+`JR2_CUSTODIAN_URL`. jr2 adds `HOME`, `TMPDIR` and the Stand-in of each Held secret that the `workspace()` lists, under
+the name of that secret. The environment holds no control Stand-in. It holds the Stand-in of a model key only when the
+`workspace()` lists that key. Thus code in `/work` cannot call the model with your key unless you allow it. jr2 records
+the use of each Turn, and a call that you did not allow cannot go around that record.
+
+The read, write and edit tools reach the same paths as the shell. A path in `/run/jr2`, or a link to one, gives the
+error of a file that the user cannot read. In a path, `~` is the `HOME` of the confined programs.
+
+The Harness process has its own `HOME`, in `/run/jr2`, from before Node starts. Thus a file that a program writes in its
+`HOME`, such as `~/.aws/config` or a module in `~/.node_modules`, does not change what the Harness process does. The
+Harness does not start when Node would look for a module outside `/opt/jr2` and `/run/jr2`.
+
+The Harness process gets only the env that jr2 composes: `harness.env`, `harness.envFrom`, the proxy and trust
+variables, and jr2's own names. It does not get the `ENV` of the image. It also does not get a name that loads code from
+a path, such as `NODE_OPTIONS`, `LD_PRELOAD` or `OPENSSL_CONF`, from any of them. Thus an image with
+`NODE_OPTIONS=--require /app/hook.js` changes the Agent's programs, as you expect, and not the Harness process. In
+`kubectl exec`, each key of `harness.envFrom` has the prefix `JR2_ENVFROM_`. The programs of the Agent see each key
+under its own name. A name that starts with `JR2_` is jr2's own: `jr2 up` refuses one in `harness.env` or in a Secret or
+a ConfigMap that `harness.envFrom` loads, and the Harness does not start with one from `harness.envFrom`.
+
+Thus code in `/work` cannot send a pick for an Agent, also when a different Agent runs it. An Agent can still try to
+change what a different Agent does with the text that it writes, such as a comment in a file. The other Agent's model
+decides each of its own picks.
+
+jr2 needs Landlock in the kernel of each node. See [Rules](#rules).
+
+## Effect secrets
+
+An Effect secret is the real value, in an Effector. The Effector reads the value and puts it into the call, or into the
+environment of a `command`. The Agent executes nothing in the Effector, and the Orchestrator never holds the value.
+[Effects](effects.md) tells which Effects run in which Effector.
+
+A `command` that names an Effect secret can run the Agent's code, such as `npm run deploy`. It runs on a copy of one
+commit of the Agent's branch, in its scratch folder. Because the command can run the Agent's code, the Effector confines
+the process of the command with Landlock. The process can read and run the system files of the image. It can use its own
+scratch folder, and read the Effect secrets that the command names. It cannot read the Effector's key, the other Effect
+secrets, the push credentials or the [Worktree](../reference/glossary.md#worktree).
+
+The Effector removes each Effect secret that it holds from what it returns, in the forms that it sent and in common
+encodings. It does not find a changed form of an Effect secret, such as a hash or a part of it.
+
+## Git credentials
+
+The [`git.credentials`](../reference/config.md#git-credentials) entries give the cluster its git credentials. The
+Orchestrator matches a Repo to an entry, and names the Secret of that entry. It never reads a git credential:
+
+- The clone credential goes into a Secret that only the Repo cache reads.
+- The push credential and the signing key go to the [Sandbox Effector](../reference/glossary.md#sandbox-effector) of the
+  Sandbox whose `push` Effect uses them.
+
+The list is also a fence. A Repo url that comes from the run input must match an entry, or the Orchestrator refuses to
+make the Sandbox. [Give the cluster git credentials](../guides/give-the-cluster-git-credentials.md) gives the fields.
+
+## The tokens
+
+jr2 makes its own tokens. `jr2 up` makes the Instance token and the signing key the first time, and keeps them in the
+`jr2-instance` Secret. Each other token comes from the signing key, so the Orchestrator stores no token for each pod.
+
+| Token          | Who holds it                                                                                         | What it lets the holder do                                                                                  |
+| -------------- | ---------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------- |
+| Instance token | You, the CLI, the [Console](../reference/glossary.md#console) when you type it, and a webhook client | Everything: start a run, read its full state, answer a Gate, cancel a run.                                  |
+| Control token  | The Custodian of one Sandbox, as a file                                                              | Read the Menu and send the picks of the Agents in that Sandbox, for the Agent that the Harness names.       |
+| Fetch token    | The Custodian of one Sandbox, as a file                                                              | Ask for a fetch of a Repo that the Sandbox mounts.                                                          |
+| Harness bearer | The Orchestrator. The Harness holds only its SHA-256 digest.                                         | Call the Harness in one pod: start a Turn, read it, stop it, set up a Worktree, and run a `command` Effect. |
+| Effector key   | The Orchestrator, and the Effector of one placement                                                  | Sign a request to that Effector, and sign its answer.                                                       |
+
+- The Instance token enters no Harness pod. The CLI reads it from the `jr2-instance` Secret over the Kubernetes API.
+  Thus Kubernetes RBAC decides who holds it.
+- A control token or a fetch token cannot answer a Gate, and cannot speak for a different pod. The Orchestrator accepts
+  each one only on its own routes. Thus an Agent cannot approve its own work, and a Sandbox cannot send a pick for an
+  Agent in a different Sandbox.
+- A Harness can check its bearer, but it cannot make a bearer. Thus an Agent cannot drive a conversation in a different
+  pod.
+
+## Open observation, guarded control
+
+Each route of the [HTTP API](../reference/http-api.md) is in one of three access bands:
+
+- [**Open**](../reference/http-api.md#band-open), with no token: the [Workflows](../reference/glossary.md#workflow), the
+  structure of each Machine, and the live runs of a Workflow with their states. An open route shows no context of a run,
+  and an [Emit](../reference/glossary.md#emit) shows only its type.
+- [**Authenticated**](../reference/http-api.md#band-authenticated), with the Instance token or a token of a Sandbox: the
+  routes of the Agent's Menu and the fetch. A token of a Sandbox reaches only its own routes, in its own pod.
+- [**Instance token only**](../reference/http-api.md#band-instanceonly): the start of a run, the full state of a run,
+  the Gates, and cancel.
+
+Thus the Console shows the Machines and the runs without a token. When you type the Instance token in the Console, it
+can also start runs and answer Gates. The Console keeps the token in the session storage of the browser tab, and forgets
+it when you close the tab.
+
+"Open" means open to each caller that can reach the Service of the Orchestrator: a port forward, a pod in the cluster,
+or an ingress. The names of the states and the types of the Emits are open.
+
+The Harness writes each Agent's conversation to its container log. The log has the prompts, the model's text and
+thinking, its tool calls and the answers to its picks. It does not write the results of the Working tools. Each person
+who can read the pod logs of the namespace can read these conversations.
+
+## Rules
+
+- A model key goes in the `secrets` list. `jr2 up` refuses a known model key in `harness.env`, or in a Secret or a
+  ConfigMap that `harness.envFrom` names.
+- A Held secret goes only over `https://`. `jr2 up` refuses `harness.provider.apiKey` when `harness.provider.baseUrl` is
+  not `https://`. A local model over plain `http://` has no key.
+- [`harness.env`](../reference/config.md#harness-env) and `harness.envFrom` put each value into the Harness container,
+  and the Agent can read it. Put a credential in the `secrets` list.
+- A file that the Harness process reads, such as the one that `AWS_CONFIG_FILE` or `GOOGLE_APPLICATION_CREDENTIALS`
+  names in `harness.env`, cannot be in `/work`, `/tmp`, `/dev/shm` or `/home/jr2`: `jr2 up` refuses it. In another
+  folder, `jr2 up` gives a warning: make sure that the user of the Sandbox cannot write the file.
+- A Held secret must name its `hosts`, unless it is a known model key with one default host.
+- A name that a `workspace()` lists must be in the `secrets` list. A name that an Effect gives to `secret()` must be in
+  it too.
+- An Effect secret has 8 characters or more.
+- The name of a Held secret is an env var name. It cannot start with `JR2_`, and it cannot be a proxy or trust variable.
+- The Orchestrator can make and change a Secret, but it cannot get or list one.
+- A changed value does not reach a live Sandbox. A new Sandbox gets it. When a Menu-only Agent uses the value, the next
+  `jr2 up` restarts the Instance Harness with it.
+- jr2 needs Landlock ABI 3 or later in the kernel of each Sandbox node: Linux 6.2 or later, or a kernel with a backport,
+  with `landlock` in its list of security modules. `jr2 up` refuses to converge while a Sandbox node has no Landlock, or
+  only ABI 1 or 2. On a node without it, a Harness or a Sandbox Effector does not start, and the provision of the
+  Sandbox fails.
+- A name in `harness.env`, or a key that `harness.envFrom` loads, cannot start with `JR2_`.
+- When a pod holds a Held secret other than the tokens of its Sandbox, `SSL_CERT_FILE` and the other trust variables
+  replace the trust store of the image. They hold the public roots, `harness.caBundle` and the CA of the Instance. Thus
+  a private CA goes in [`harness.caBundle`](../reference/config.md#harness-cabundle), not in the image.
+- `jr2 up` makes the CA of the Instance one time, in the `jr2-held-ca` Secret, and never changes it. No pod mounts that
+  Secret. When you delete it, the next `jr2 up` makes a new CA.
+
+## Read next
+
+- [Use a local model](../guides/use-a-local-model.md)
+- [Use a hosted provider or a gateway](../guides/use-a-hosted-provider.md)
+- [Give the cluster git credentials](../guides/give-the-cluster-git-credentials.md)
+- [Run a command or call a URL with an Effect](../guides/run-a-command-or-call-a-url.md)
+- [Open the Console](../guides/open-the-console.md)
+- [Architecture](architecture.md) and [Effects](effects.md)
+- The [`secrets`](../reference/config.md#secrets), [`git`](../reference/config.md#git) and
+  [`harness`](../reference/config.md#harness) fields of `jr2.config.ts`,
+  [`secret()`](../reference/authoring-surface/secret.md), and the [HTTP API](../reference/http-api.md)
+- [ADR-0013](../reference/decisions/0013-the-agent-reaches-its-machine-through-a-container-it-cannot-read.md),
+  [ADR-0014](../reference/decisions/0014-observation-is-open-run-state-is-not.md),
+  [ADR-0023](../reference/decisions/0023-the-harness-prints-the-conversation.md),
+  [ADR-0032](../reference/decisions/0032-the-console-unlocks-with-the-instance-token.md),
+  [ADR-0051](../reference/decisions/0051-a-repo-is-a-slot-on-the-workspace-and-a-cache-on-the-node.md),
+  [ADR-0058](../reference/decisions/0058-a-harness-answers-the-orchestrator-alone.md),
+  [ADR-0059](../reference/decisions/0059-a-harness-holds-stand-ins-and-the-custodian-holds-the-keys.md),
+  [ADR-0068](../reference/decisions/0068-an-effector-holds-an-effects-secrets-and-signs-its-wire.md),
+  [ADR-0076](../reference/decisions/0076-an-agents-control-credential-lives-only-where-no-workspace-code-runs.md) and
+  [ADR-0077](../reference/decisions/0077-jr2-requires-landlock-on-every-sandbox-node.md)