@lanes-sh/link 0.8.0 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -25
- package/instructions/skills/lanes-link/SKILL.md +12 -12
- package/package.json +1 -1
- package/src/auth/oauth/store.ts +21 -5
- package/src/cli/accepts.ts +6 -5
- package/src/cli/brand.ts +20 -15
- package/src/cli/callback-page.ts +8 -7
- package/src/cli/commands/connect/index.ts +2 -1
- package/src/cli/commands/connect/settle.ts +12 -4
- package/src/cli/commands/connection.ts +2 -1
- package/src/cli/commands/identity.ts +4 -4
- package/src/cli/commands/knowledge/index.ts +9 -16
- package/src/cli/commands/knowledge/migrate.ts +9 -4
- package/src/cli/commands/knowledge/show.ts +14 -9
- package/src/cli/commands/operate/migrate.ts +5 -2
- package/src/cli/commands/operate/pair.ts +1 -1
- package/src/cli/commands/operate/serve.ts +1 -1
- package/src/cli/commands/owner/assets.ts +2 -2
- package/src/cli/commands/owner/entities.ts +2 -2
- package/src/cli/commands/owner/memory.ts +2 -2
- package/src/cli/commands/owner/tasks.ts +2 -2
- package/src/cli/commands/owner/vault.ts +3 -3
- package/src/cli/commands/profile/disposition.ts +236 -0
- package/src/cli/commands/profile/removal.ts +100 -13
- package/src/cli/commands/profile/remove.ts +67 -6
- package/src/cli/commands/profile.ts +33 -6
- package/src/cli/commands/secrets.ts +4 -4
- package/src/cli/commands/update-migration.ts +54 -0
- package/src/cli/commands/update.ts +36 -23
- package/src/cli/config-edit.ts +29 -8
- package/src/cli/config-repair-sweep.ts +183 -0
- package/src/cli/config-repair.ts +90 -141
- package/src/cli/config-templates.ts +26 -24
- package/src/cli/contract3-credentials.ts +294 -0
- package/src/cli/contract3-data.ts +143 -209
- package/src/cli/contract3-layout.ts +46 -0
- package/src/cli/contract3-shape.ts +34 -8
- package/src/cli/contract3.ts +141 -24
- package/src/cli/contract4-credentials.ts +207 -0
- package/src/cli/contract4-data.ts +399 -0
- package/src/cli/contract4-rename.ts +73 -0
- package/src/cli/contract4-yaml.ts +223 -0
- package/src/cli/contract4.ts +349 -0
- package/src/cli/identity.ts +44 -26
- package/src/cli/main.ts +6 -1
- package/src/cli/migrate-move.ts +166 -0
- package/src/cli/migrate-plan.ts +3 -3
- package/src/cli/publish.ts +1 -5
- package/src/cli/runtime/open.ts +5 -5
- package/src/cli/runtime/select.ts +2 -11
- package/src/cli/runtime/stores.ts +16 -11
- package/src/cli/runtime/vault.ts +2 -2
- package/src/cli/usage.ts +5 -1
- package/src/cli/workspace-migrate.ts +32 -11
- package/src/connectivity/manifest/provider.ts +31 -12
- package/src/connectivity/transports/imap/parser.ts +70 -9
- package/src/deployments/adapters/filesystem.ts +18 -3
- package/src/deployments/deploy.ts +5 -5
- package/src/deployments/gcp/bucket.ts +42 -6
- package/src/deployments/knowledge.ts +9 -4
- package/src/deployments/target.ts +28 -7
- package/src/deployments/upload.ts +39 -30
- package/src/profile/connections.ts +13 -1
- package/src/profile/deployments.ts +86 -8
- package/src/profile/index.ts +5 -1
- package/src/profile/knowledge.ts +18 -5
- package/src/profile/layout.ts +147 -71
- package/src/profile/load.ts +53 -17
- package/src/profile/schema.ts +11 -2
- package/src/profile/testing.ts +45 -10
- package/src/profile/workspace.ts +66 -30
- package/src/providers/assets/provider.ts +6 -6
- package/src/providers/entities/provider.ts +6 -6
- package/src/providers/entities/writes.ts +1 -1
- package/src/providers/identity/provider.ts +1 -1
- package/src/providers/memory/provider.ts +6 -6
- package/src/providers/setup/provider.ts +3 -3
- package/src/providers/skills/provider.ts +2 -2
- package/src/providers/tasks/provider.ts +6 -6
- package/src/providers/vault/provider.ts +1 -1
- package/src/registry/registry.ts +1 -1
- package/src/server/endpoint.ts +4 -0
- package/src/server/harness.ts +1 -1
- package/src/server/mcp/instructions.ts +21 -21
- package/src/server/mcp/routing.ts +3 -3
- package/src/server/mcp/tools.ts +16 -3
- package/src/server/mcp/visibility.ts +56 -4
- package/src/stores/blobs/conformance.ts +19 -0
- package/src/stores/state/index.ts +76 -10
- package/src/stores/state/testing.ts +5 -1
package/src/cli/runtime/open.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { BearerAuthenticator, ownerPrincipal } from '#auth';
|
|
2
2
|
import type { SecretStore } from '#secrets';
|
|
3
3
|
import type { AuditReader } from '#audit';
|
|
4
|
-
import type
|
|
4
|
+
import { DISCOVERY_NAMESPACE, type RuntimeState } from '#stores/state';
|
|
5
5
|
import type { BlobStore } from '#stores/blobs';
|
|
6
6
|
import type { AnyConnector, ProviderManifest } from '#connectivity';
|
|
7
7
|
import { RateLimiter, allowedConnections } from '#policy';
|
|
@@ -127,7 +127,7 @@ export async function openRuntime(
|
|
|
127
127
|
// own roots on the target's own storage and are untouched.
|
|
128
128
|
const knowledge = await openKnowledge(adapters, credentials, options.fetch);
|
|
129
129
|
const storage = knowledge ? routeBlobStore(storageFor(), knowledgeRoutes(knowledge)) : storageFor();
|
|
130
|
-
const state = openState(storageFor);
|
|
130
|
+
const state = openState(storageFor, config.instance.profile);
|
|
131
131
|
|
|
132
132
|
// The durable log, plus any copies the target declares. `sink` is what
|
|
133
133
|
// dispatch writes to; `audit` is what `tail` and `verify` read, and those are
|
|
@@ -148,10 +148,10 @@ export async function openRuntime(
|
|
|
148
148
|
// `EMPTY_SKILLS` when none is granted: a profile that denies `skills.*` has
|
|
149
149
|
// no store to open, and handing it the workspace root instead would serve
|
|
150
150
|
// every other profile's procedures.
|
|
151
|
-
const skillsConnection = soleGrantFor(config, '
|
|
151
|
+
const skillsConnection = soleGrantFor(config, 'lanes_skills');
|
|
152
152
|
const skills =
|
|
153
153
|
knowledge?.skills ??
|
|
154
|
-
(skillsConnection === undefined ? undefined : skillStore(storageFor, skillsConnection));
|
|
154
|
+
(skillsConnection === undefined ? undefined : skillStore(storageFor, config.instance.profile, skillsConnection));
|
|
155
155
|
|
|
156
156
|
// The vault's own store, beside the credential store and never it: a separate
|
|
157
157
|
// document, a separate key, and a separate environment variable
|
|
@@ -269,7 +269,7 @@ export async function openRuntime(
|
|
|
269
269
|
}
|
|
270
270
|
}
|
|
271
271
|
|
|
272
|
-
const cached = await state.kv.get(
|
|
272
|
+
const cached = await state.kv.get(DISCOVERY_NAMESPACE, entry.manifest.id);
|
|
273
273
|
if (cached) {
|
|
274
274
|
try {
|
|
275
275
|
registry.setDiscovered(entry.manifest.id, JSON.parse(cached));
|
|
@@ -126,21 +126,12 @@ export async function resolveProfileOnly(
|
|
|
126
126
|
* inside Google's network. Opening a full runtime for either would fail on the
|
|
127
127
|
* part that is not needed.
|
|
128
128
|
*/
|
|
129
|
-
export async function openSecretStoreFor(
|
|
130
|
-
config: Config,
|
|
131
|
-
root: string,
|
|
132
|
-
target: string,
|
|
133
|
-
): Promise<SecretStore> {
|
|
129
|
+
export async function openSecretStoreFor(root: string, target: string): Promise<SecretStore> {
|
|
134
130
|
// Resolved here rather than taken from the caller, so `secrets push --from
|
|
135
131
|
// local --to cloud` can hold two targets that live in two different workspaces
|
|
136
132
|
// without the caller having to follow either pointer itself.
|
|
137
133
|
const resolved = await openTarget(root, target);
|
|
138
|
-
return openSecrets({
|
|
139
|
-
declared: resolved.declared,
|
|
140
|
-
config,
|
|
141
|
-
root: resolved.workspaceRoot,
|
|
142
|
-
target,
|
|
143
|
-
});
|
|
134
|
+
return openSecrets({ declared: resolved.declared, root: resolved.workspaceRoot, target });
|
|
144
135
|
}
|
|
145
136
|
|
|
146
137
|
/**
|
|
@@ -33,21 +33,26 @@ export const EMPTY_SKILL_STORE: BlobStore = {
|
|
|
33
33
|
/**
|
|
34
34
|
* Where a connection's skills live, in either workspace.
|
|
35
35
|
*
|
|
36
|
-
* `
|
|
37
|
-
* prefix. Going through the store rather than a filesystem path is what gives a
|
|
36
|
+
* `profiles/<profile>/skills.d/<connection>/`, and deployed the same key under
|
|
37
|
+
* the bucket prefix. Going through the store rather than a filesystem path is what gives a
|
|
38
38
|
* deployment skills at all — a path is baked into a container image at build
|
|
39
39
|
* time and an object key is not, so before ADR-014 a deployed instance could
|
|
40
40
|
* only ever serve the skills that existed when its image was built.
|
|
41
41
|
*
|
|
42
|
-
* **Per connection**, which is the
|
|
43
|
-
* gating `skills.<name>` was the whole isolation story while
|
|
44
|
-
* shared (ADR-012 §1), and it is a weak one: it decides who may
|
|
45
|
-
* procedure, not who may read that it exists or what it says. ADR-030
|
|
46
|
-
* bytes per profile; ADR-059 made them per connection, so two
|
|
47
|
-
* one skills connection
|
|
48
|
-
*
|
|
42
|
+
* **Per profile and per connection**, which is the fourth answer this question
|
|
43
|
+
* has had. Policy gating `skills.<name>` was the whole isolation story while
|
|
44
|
+
* the bytes were shared (ADR-012 §1), and it is a weak one: it decides who may
|
|
45
|
+
* *run* a procedure, not who may read that it exists or what it says. ADR-030
|
|
46
|
+
* made the bytes per profile; ADR-059 made them per connection instead, so two
|
|
47
|
+
* profiles granting one skills connection shared a set; ADR-066 puts the
|
|
48
|
+
* profile back in front of it, so they do not. The connection stays because a
|
|
49
|
+
* profile may still hold more than one set.
|
|
49
50
|
*/
|
|
50
|
-
export function skillStore(
|
|
51
|
-
|
|
51
|
+
export function skillStore(
|
|
52
|
+
storage: StorageFactory,
|
|
53
|
+
profile: string,
|
|
54
|
+
connection: string,
|
|
55
|
+
): BlobStore {
|
|
56
|
+
return storage(layout.skills(profile, connection));
|
|
52
57
|
}
|
|
53
58
|
|
package/src/cli/runtime/vault.ts
CHANGED
|
@@ -40,12 +40,12 @@ export function openVault(
|
|
|
40
40
|
// The vault connection this profile grants (ADR-059). `main` when it grants
|
|
41
41
|
// none, which keeps a profile that denied the vault opening against the same
|
|
42
42
|
// document every other profile uses rather than inventing a second one.
|
|
43
|
-
const connection = soleGrantFor(config, '
|
|
43
|
+
const connection = soleGrantFor(config, 'lanes_vault') ?? 'main';
|
|
44
44
|
|
|
45
45
|
switch (vault.adapter) {
|
|
46
46
|
case 'file':
|
|
47
47
|
return createFileVaultStore({
|
|
48
|
-
path: workspacePath(root, vault.path ?? layout.vault(connection)),
|
|
48
|
+
path: workspacePath(root, vault.path ?? layout.vault(config.instance.profile, connection)),
|
|
49
49
|
});
|
|
50
50
|
|
|
51
51
|
case 'secret':
|
package/src/cli/usage.ts
CHANGED
|
@@ -61,7 +61,11 @@ ${style.bold('Profiles')}
|
|
|
61
61
|
rest are copied from a sibling profile
|
|
62
62
|
${PROGRAM} profile list [--json]
|
|
63
63
|
${PROGRAM} profile remove <name> [--workspace <name>] [--dry-run] [--yes] [--json]
|
|
64
|
-
|
|
64
|
+
[--delete-data | --migrate-to <profile>]
|
|
65
|
+
the profile and its own token. Say which of
|
|
66
|
+
--delete-data or --migrate-to for its memory,
|
|
67
|
+
tasks, assets and skills — there is no default.
|
|
68
|
+
Accounts outlive it; disconnect removes those.
|
|
65
69
|
|
|
66
70
|
${style.bold('Workspaces')}
|
|
67
71
|
${PROGRAM} workspace list [--urls] every workspace this one knows
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { migrateToContract4, type Contract4Migration } from './contract4.ts';
|
|
1
2
|
import { migrateToContract3, needsContract3, type Contract3Migration } from './contract3.ts';
|
|
2
3
|
import { readSession } from '#auth/lanes/session.ts';
|
|
3
4
|
import { parseDocument } from 'yaml';
|
|
@@ -18,6 +19,7 @@ import {
|
|
|
18
19
|
} from '#profile';
|
|
19
20
|
import { ConfigDocument } from './config-edit.ts';
|
|
20
21
|
import { hoist, summarise } from './migrate-plan.ts';
|
|
22
|
+
import { C3 } from './contract3-layout.ts';
|
|
21
23
|
|
|
22
24
|
/**
|
|
23
25
|
* Contract 1 → 2: the target moves out of the profile and into the workspace.
|
|
@@ -64,7 +66,7 @@ export interface WorkspaceMigration {
|
|
|
64
66
|
/** Whether this workspace still holds anything at contract 1. */
|
|
65
67
|
export async function needsMigration(workspaceRoot: string): Promise<boolean> {
|
|
66
68
|
for (const profile of await listProfiles(workspaceRoot)) {
|
|
67
|
-
const text = await readWorkspaceFile(workspaceFiles(workspaceRoot),
|
|
69
|
+
const text = await readWorkspaceFile(workspaceFiles(workspaceRoot), C3.profile(profile));
|
|
68
70
|
if (text === null) continue;
|
|
69
71
|
try {
|
|
70
72
|
if (isLegacyProfile(parseDocument(text).toJSON())) return true;
|
|
@@ -93,7 +95,10 @@ export async function migrateWorkspace(
|
|
|
93
95
|
[];
|
|
94
96
|
|
|
95
97
|
for (const profile of names) {
|
|
96
|
-
|
|
98
|
+
// The contract-1 path, frozen. `open` resolves the live layout, which is
|
|
99
|
+
// contract 4's — so this read a profile that does not exist yet and the
|
|
100
|
+
// migration refused a workspace it was supposed to move.
|
|
101
|
+
const document = await ConfigDocument.openKey(workspaceRoot, C3.profile(profile));
|
|
97
102
|
const raw = document.toJSON();
|
|
98
103
|
if (!isLegacyProfile(raw)) continue;
|
|
99
104
|
|
|
@@ -126,10 +131,10 @@ export async function migrateWorkspace(
|
|
|
126
131
|
changes.push(
|
|
127
132
|
entry.at !== undefined
|
|
128
133
|
? `workspaces.${name}: pointer to ${entry.at}`
|
|
129
|
-
: `targets.${name}: declared in ${
|
|
134
|
+
: `targets.${name}: declared in ${C3.workspace}`,
|
|
130
135
|
);
|
|
131
136
|
}
|
|
132
|
-
for (const { profile } of legacy) changes.push(
|
|
137
|
+
for (const { profile } of legacy) changes.push(`${C3.profile(profile)}: targets: removed, contract: 2`);
|
|
133
138
|
|
|
134
139
|
if (!options.apply) {
|
|
135
140
|
return {
|
|
@@ -170,7 +175,7 @@ export async function migrateWorkspace(
|
|
|
170
175
|
// wrote, and `check` refuses anything either step leaves broken.
|
|
171
176
|
await writeWorkspaceFile(
|
|
172
177
|
workspaceFiles(workspaceRoot),
|
|
173
|
-
|
|
178
|
+
C3.profile(profile),
|
|
174
179
|
document.toString(),
|
|
175
180
|
);
|
|
176
181
|
}
|
|
@@ -190,6 +195,7 @@ export interface ContractMigration {
|
|
|
190
195
|
readonly legacy: WorkspaceMigration | null;
|
|
191
196
|
/** The contract 2 → 3 half, when this workspace needed one. */
|
|
192
197
|
readonly contract3: Contract3Migration | null;
|
|
198
|
+
readonly contract4: Contract4Migration | null;
|
|
193
199
|
/** Every profile either half rewrote, deduplicated. */
|
|
194
200
|
readonly profiles: readonly string[];
|
|
195
201
|
/** Targets written into the registry by the contract 1 → 2 half. */
|
|
@@ -236,7 +242,7 @@ export interface ContractMigration {
|
|
|
236
242
|
*/
|
|
237
243
|
export async function migrateToCurrentContract(
|
|
238
244
|
workspaceRoot: string,
|
|
239
|
-
options: { apply: boolean; subject?: string } = { apply: true },
|
|
245
|
+
options: { apply: boolean; subject?: string; target?: string } = { apply: true },
|
|
240
246
|
): Promise<ContractMigration> {
|
|
241
247
|
const legacy = (await needsMigration(workspaceRoot))
|
|
242
248
|
? await migrateWorkspace(workspaceRoot, { apply: options.apply })
|
|
@@ -249,14 +255,29 @@ export async function migrateToCurrentContract(
|
|
|
249
255
|
...(subject === undefined ? {} : { subject }),
|
|
250
256
|
});
|
|
251
257
|
|
|
258
|
+
// In sequence, not in parallel: contract 4 moves what contract 3 produced, so
|
|
259
|
+
// it has to run against the tree the previous step left. With `apply: false`
|
|
260
|
+
// it sees the unmigrated shape and reports only what it can see from here —
|
|
261
|
+
// which is the honest preview, and why the count is not promised.
|
|
262
|
+
const contract4 = await migrateToContract4(workspaceRoot, {
|
|
263
|
+
apply: options.apply,
|
|
264
|
+
...(options.target === undefined ? {} : { target: options.target }),
|
|
265
|
+
});
|
|
266
|
+
|
|
252
267
|
return {
|
|
253
268
|
workspaceRoot,
|
|
254
269
|
legacy: legacy !== null && !legacy.alreadyCurrent ? legacy : null,
|
|
255
270
|
contract3: contract3.alreadyCurrent ? null : contract3,
|
|
256
|
-
|
|
271
|
+
contract4: contract4.alreadyCurrent ? null : contract4,
|
|
272
|
+
profiles: [
|
|
273
|
+
...new Set([...(legacy?.profiles ?? []), ...contract3.profiles, ...contract4.profiles]),
|
|
274
|
+
],
|
|
257
275
|
targets: legacy?.targets ?? [],
|
|
258
|
-
changes: [...(legacy?.changes ?? []), ...contract3.changes],
|
|
259
|
-
alreadyCurrent:
|
|
276
|
+
changes: [...(legacy?.changes ?? []), ...contract3.changes, ...contract4.changes],
|
|
277
|
+
alreadyCurrent:
|
|
278
|
+
(legacy === null || legacy.alreadyCurrent) &&
|
|
279
|
+
contract3.alreadyCurrent &&
|
|
280
|
+
contract4.alreadyCurrent,
|
|
260
281
|
};
|
|
261
282
|
}
|
|
262
283
|
|
|
@@ -274,7 +295,7 @@ async function writeRegistry(
|
|
|
274
295
|
registry: Record<string, WorkspaceTarget>,
|
|
275
296
|
): Promise<void> {
|
|
276
297
|
const files = workspaceFiles(workspaceRoot);
|
|
277
|
-
const text = (await readWorkspaceFile(files,
|
|
298
|
+
const text = (await readWorkspaceFile(files, C3.workspace)) ?? `contract: ${SUPPORTED_CONTRACT}\n`;
|
|
278
299
|
const document = parseDocument(text);
|
|
279
300
|
const current = (document.toJSON() ?? {}) as {
|
|
280
301
|
workspaces?: Record<string, WorkspaceTarget>;
|
|
@@ -318,7 +339,7 @@ async function writeRegistry(
|
|
|
318
339
|
document.deleteIn(['deployments']);
|
|
319
340
|
document.deleteIn(['default_target']);
|
|
320
341
|
|
|
321
|
-
await writeWorkspaceFile(files,
|
|
342
|
+
await writeWorkspaceFile(files, C3.workspace, String(document));
|
|
322
343
|
}
|
|
323
344
|
|
|
324
345
|
function describe(
|
|
@@ -76,25 +76,44 @@ export const providerManifestSchema = z.object({
|
|
|
76
76
|
export type ProviderManifest = z.infer<typeof providerManifestSchema>;
|
|
77
77
|
|
|
78
78
|
/**
|
|
79
|
-
*
|
|
79
|
+
* The owner layer's provider ids — Lanes' own surfaces.
|
|
80
|
+
*
|
|
81
|
+
* **`lanes_` on each, which is what stops them needing to be reserved.** They
|
|
82
|
+
* were `memory`, `tasks`, `assets`, `skills`, `vault`, `entities` — six of the
|
|
83
|
+
* most obvious words a vendor manifest might want, held back from every
|
|
84
|
+
* operator so the built-ins could have them. `buildRegistry` registers these
|
|
85
|
+
* before `PROVIDERS`, so a manifest claiming one threw at startup rather than
|
|
86
|
+
* being shadowed (ADR-051); the reservation is what made that a refusal instead
|
|
87
|
+
* of a collision. Prefixed, there is nothing to reserve: an operator's own
|
|
88
|
+
* `memory` connector is now a legal thing to declare.
|
|
89
|
+
*
|
|
90
|
+
* It is also the shape the vendor-qualified providers already use —
|
|
91
|
+
* `google_tasks`, `gmail_imap`, `icloud_mail` — and it reads the same way: the
|
|
92
|
+
* half before the underscore says whose surface this is.
|
|
80
93
|
*
|
|
81
94
|
* The order is read: `#server/mcp`'s instructions emit one paragraph per
|
|
82
95
|
* reachable id in this sequence, so it is the order an agent meets them in.
|
|
83
|
-
* `
|
|
84
|
-
* beside `
|
|
85
|
-
* and the instructions collapse them into one paragraph when
|
|
96
|
+
* `lanes_entities` is appended rather than inserted alphabetically so that it
|
|
97
|
+
* lands beside `lanes_identity`: the two answer the same question about
|
|
98
|
+
* different people, and the instructions collapse them into one paragraph when
|
|
99
|
+
* both are reachable.
|
|
86
100
|
*/
|
|
87
101
|
export const RESERVED_PROVIDER_IDS: readonly string[] = [
|
|
88
|
-
'
|
|
89
|
-
'
|
|
90
|
-
'
|
|
91
|
-
'
|
|
92
|
-
'
|
|
93
|
-
'
|
|
94
|
-
'
|
|
95
|
-
'
|
|
102
|
+
'lanes_memory',
|
|
103
|
+
'lanes_tasks',
|
|
104
|
+
'lanes_assets',
|
|
105
|
+
'lanes_skills',
|
|
106
|
+
'lanes_vault',
|
|
107
|
+
'lanes_setup',
|
|
108
|
+
'lanes_identity',
|
|
109
|
+
'lanes_entities',
|
|
96
110
|
];
|
|
97
111
|
|
|
112
|
+
/** Old id to new, for the contract-4 migration and for a refusal that names it. */
|
|
113
|
+
export const RENAMED_OWNER_PROVIDERS: ReadonlyMap<string, string> = new Map(
|
|
114
|
+
RESERVED_PROVIDER_IDS.map((id) => [id.slice('lanes_'.length), id]),
|
|
115
|
+
);
|
|
116
|
+
|
|
98
117
|
/**
|
|
99
118
|
* Validate a manifest, with the cross-field rules the schema alone cannot
|
|
100
119
|
* express.
|
|
@@ -16,6 +16,30 @@
|
|
|
16
16
|
const CR = 0x0d;
|
|
17
17
|
const LF = 0x0a;
|
|
18
18
|
|
|
19
|
+
/**
|
|
20
|
+
* The most this will accumulate before refusing a response.
|
|
21
|
+
*
|
|
22
|
+
* A literal's length is a number the *server* writes — `{1234}` — and the
|
|
23
|
+
* reader's job is to wait until that many bytes have arrived. With no ceiling
|
|
24
|
+
* that is an allocation an upstream decides the size of, and the upstream is
|
|
25
|
+
* not always one this endpoint chose: a connector names its own host, so a
|
|
26
|
+
* connection pointed at a hostile or compromised server could announce a
|
|
27
|
+
* literal of any size and be believed.
|
|
28
|
+
*
|
|
29
|
+
* Sixty-four mebibytes because that is already the ceiling on the other side of
|
|
30
|
+
* the same journey — `MAX_UPLOAD_BYTES` in `server/attachments.ts` — and a
|
|
31
|
+
* message larger than the largest attachment this endpoint will accept is not
|
|
32
|
+
* one it can do anything useful with. Every mail host's own limit is well below
|
|
33
|
+
* it, so this is never what refuses a legitimate read.
|
|
34
|
+
*
|
|
35
|
+
* Applied twice, and both are needed. The announced length is refused up front,
|
|
36
|
+
* so an absurd number costs nothing rather than being discovered after the
|
|
37
|
+
* bytes arrive. The accumulated buffer is refused too, because one response may
|
|
38
|
+
* announce several literals and a server that never completes a response would
|
|
39
|
+
* otherwise grow this without ever announcing anything unreasonable.
|
|
40
|
+
*/
|
|
41
|
+
export const MAX_RESPONSE_BYTES = 64 * 1024 * 1024;
|
|
42
|
+
|
|
19
43
|
/** A parsed element of a response. */
|
|
20
44
|
export type ImapToken =
|
|
21
45
|
| { readonly kind: 'atom'; readonly value: string }
|
|
@@ -32,27 +56,56 @@ export type ImapToken =
|
|
|
32
56
|
*/
|
|
33
57
|
export class ResponseAssembler {
|
|
34
58
|
#buffer = new Uint8Array(0);
|
|
35
|
-
|
|
59
|
+
/** Bytes in use. The buffer is grown ahead of this and is not a length. */
|
|
60
|
+
#length = 0;
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Take a chunk off the socket.
|
|
64
|
+
*
|
|
65
|
+
* Capacity doubles rather than growing to fit. It used to allocate exactly
|
|
66
|
+
* `held + chunk` and copy everything across on every chunk, which is
|
|
67
|
+
* quadratic in the size of a response — a thirty-megabyte message arriving in
|
|
68
|
+
* sixty-four-kilobyte pieces copied several gigabytes to assemble, and that
|
|
69
|
+
* was the *legitimate* case. Doubling makes each byte move a constant number
|
|
70
|
+
* of times.
|
|
71
|
+
*/
|
|
36
72
|
push(chunk: Uint8Array): void {
|
|
37
|
-
const
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
73
|
+
const needed = this.#length + chunk.length;
|
|
74
|
+
if (needed > MAX_RESPONSE_BYTES) {
|
|
75
|
+
throw new Error(
|
|
76
|
+
`The server sent more than ${MAX_RESPONSE_BYTES} bytes without completing a response.`,
|
|
77
|
+
);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
if (needed > this.#buffer.length) {
|
|
81
|
+
let capacity = Math.max(this.#buffer.length, 8192);
|
|
82
|
+
while (capacity < needed) capacity *= 2;
|
|
83
|
+
|
|
84
|
+
const grown = new Uint8Array(Math.min(capacity, MAX_RESPONSE_BYTES));
|
|
85
|
+
grown.set(this.#buffer.subarray(0, this.#length));
|
|
86
|
+
this.#buffer = grown;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
this.#buffer.set(chunk, this.#length);
|
|
90
|
+
this.#length = needed;
|
|
41
91
|
}
|
|
42
92
|
|
|
43
93
|
/** The next complete response, or undefined while more bytes are needed. */
|
|
44
94
|
next(): Uint8Array | undefined {
|
|
45
|
-
const end = completeResponseEnd(this.#buffer);
|
|
95
|
+
const end = completeResponseEnd(this.#buffer.subarray(0, this.#length));
|
|
46
96
|
if (end === undefined) return undefined;
|
|
47
97
|
|
|
48
|
-
|
|
49
|
-
|
|
98
|
+
// A copy, not a view: the remainder is shifted down in place below, which
|
|
99
|
+
// would otherwise rewrite the bytes underneath the response just returned.
|
|
100
|
+
const response = this.#buffer.slice(0, end);
|
|
101
|
+
this.#buffer.copyWithin(0, end, this.#length);
|
|
102
|
+
this.#length -= end;
|
|
50
103
|
return response;
|
|
51
104
|
}
|
|
52
105
|
|
|
53
106
|
/** Whatever has arrived but does not yet form a response. For diagnostics. */
|
|
54
107
|
get pending(): number {
|
|
55
|
-
return this.#
|
|
108
|
+
return this.#length;
|
|
56
109
|
}
|
|
57
110
|
}
|
|
58
111
|
|
|
@@ -67,6 +120,14 @@ function completeResponseEnd(buffer: Uint8Array): number | undefined {
|
|
|
67
120
|
const announced = literalLength(buffer, cursor, crlf);
|
|
68
121
|
if (announced === undefined) return crlf + 2;
|
|
69
122
|
|
|
123
|
+
// Refused on the announcement rather than after the bytes turn up: the
|
|
124
|
+
// number is the server's, and believing an absurd one means waiting for it.
|
|
125
|
+
if (announced > MAX_RESPONSE_BYTES) {
|
|
126
|
+
throw new Error(
|
|
127
|
+
`The server announced a ${announced}-byte literal, over the ${MAX_RESPONSE_BYTES}-byte limit.`,
|
|
128
|
+
);
|
|
129
|
+
}
|
|
130
|
+
|
|
70
131
|
const afterLiteral = crlf + 2 + announced;
|
|
71
132
|
if (buffer.length < afterLiteral) return undefined;
|
|
72
133
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { mkdir, readdir, readFile, rename, rm, rmdir, stat, writeFile } from 'node:fs/promises';
|
|
2
2
|
import { existsSync } from 'node:fs';
|
|
3
3
|
import { dirname, join, relative, resolve, sep } from 'node:path';
|
|
4
|
-
import type
|
|
4
|
+
import { containedKey, type BlobKey, type BlobMetadata, type BlobStore } from '#stores/blobs';
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
7
|
* Filesystem blob store — the `local` target's adapter.
|
|
@@ -54,12 +54,27 @@ export function createFilesystemBlobStore(options: FilesystemBlobStoreOptions):
|
|
|
54
54
|
* this adapter is also usable directly, and a containment check belongs at
|
|
55
55
|
* the point where a path actually becomes a filesystem operation. Defence in
|
|
56
56
|
* depth is cheap; a provider escaping its directory is not.
|
|
57
|
+
*
|
|
58
|
+
* **The rule is `containedKey`'s, not this file's.** It used to be a copy,
|
|
59
|
+
* and the copy did not agree: it tested `rel.startsWith('..')` where it meant
|
|
60
|
+
* "the first segment is `..`", so a key beginning with two dots and
|
|
61
|
+
* continuing — an ordinary name — was refused here and accepted by every
|
|
62
|
+
* other adapter. That is the divergence `conformance.ts` exists to prevent,
|
|
63
|
+
* and it was reachable from an ordinary argument, because a provider passes
|
|
64
|
+
* a caller's name straight through as a key.
|
|
65
|
+
*
|
|
66
|
+
* Kept as a resolve-then-join rather than deferring the whole path: the
|
|
67
|
+
* shared rule answers "does this land inside", and this adapter still has to
|
|
68
|
+
* turn the answer into a filesystem path.
|
|
57
69
|
*/
|
|
58
70
|
const pathFor = (key: BlobKey): string => {
|
|
59
|
-
const resolved = resolve(root, key);
|
|
71
|
+
const resolved = resolve(root, containedKey(key));
|
|
60
72
|
const rel = relative(root, resolved);
|
|
61
73
|
|
|
62
|
-
|
|
74
|
+
// Unreachable once `containedKey` has answered, and kept because this is
|
|
75
|
+
// the line where a path becomes a filesystem operation: a future change to
|
|
76
|
+
// either side should fail here rather than escape.
|
|
77
|
+
if (rel === '' || rel === '..' || rel.startsWith(`..${sep}`)) {
|
|
63
78
|
throw new Error(`Blob key resolves outside the store root: ${key}`);
|
|
64
79
|
}
|
|
65
80
|
return resolved;
|
|
@@ -14,7 +14,7 @@ import { recordDeployment, type DeploymentRecord } from './record.ts';
|
|
|
14
14
|
import { printSteps, runSteps } from './steps.ts';
|
|
15
15
|
import { driverFor } from './drivers.ts';
|
|
16
16
|
import { prepareSecrets, readableRefs, rotatableRefs } from './prepare.ts';
|
|
17
|
-
import { repairOwnerLayer } from '#cli/config-repair.ts';
|
|
17
|
+
import { repairOwnerLayer } from '#cli/config-repair-sweep.ts';
|
|
18
18
|
import { migrateToCurrentContract } from '#cli/workspace-migrate.ts';
|
|
19
19
|
import { deployedWorkspace, uploadWorkspace } from './upload.ts';
|
|
20
20
|
import { servingProfiles } from './serving.ts';
|
|
@@ -229,7 +229,7 @@ export async function deploy(flags: DeployFlags): Promise<void> {
|
|
|
229
229
|
await runSteps(driver, provision);
|
|
230
230
|
}
|
|
231
231
|
|
|
232
|
-
const credentials = await openSecretStoreFor(
|
|
232
|
+
const credentials = await openSecretStoreFor(resolution.workspaceRoot, target);
|
|
233
233
|
const prepared = await prepareSecrets({
|
|
234
234
|
config,
|
|
235
235
|
declared,
|
|
@@ -288,7 +288,7 @@ export async function deploy(flags: DeployFlags): Promise<void> {
|
|
|
288
288
|
// bucket at contract 2 — and running it after the upload is the same as not
|
|
289
289
|
// running it, because the upload writes contract-3 profiles and
|
|
290
290
|
// `needsContract3` reads the profiles.
|
|
291
|
-
await migrateToCurrentContract(workspace, { apply: true });
|
|
291
|
+
await migrateToCurrentContract(workspace, { apply: true, target });
|
|
292
292
|
|
|
293
293
|
// Before the rollout, so the revision that comes up finds a config to read.
|
|
294
294
|
// Uploading after would leave a window where the service is serving and the
|
|
@@ -308,7 +308,7 @@ export async function deploy(flags: DeployFlags): Promise<void> {
|
|
|
308
308
|
|
|
309
309
|
// Again for what the upload put there: a newly created bucket gets its
|
|
310
310
|
// profiles written here for the first time. Idempotent — one listing.
|
|
311
|
-
await migrateToCurrentContract(workspace, { apply: true });
|
|
311
|
+
await migrateToCurrentContract(workspace, { apply: true, target });
|
|
312
312
|
|
|
313
313
|
// Where this deployment lives, in both registries — see `record.ts`. The
|
|
314
314
|
// declaration has to land before the revision boots, so it goes here rather
|
|
@@ -374,7 +374,7 @@ async function migrateTargetWorkspace(target: string, apply: boolean): Promise<b
|
|
|
374
374
|
const workspace = await resolveTargetWorkspace(root, target).catch(() => null);
|
|
375
375
|
if (workspace === null || workspace === root) return true;
|
|
376
376
|
|
|
377
|
-
const migrated = await migrateToCurrentContract(workspace, { apply });
|
|
377
|
+
const migrated = await migrateToCurrentContract(workspace, { apply, target });
|
|
378
378
|
if (migrated.alreadyCurrent) return true;
|
|
379
379
|
|
|
380
380
|
heading(apply ? 'Migrated' : 'Would migrate');
|
|
@@ -51,6 +51,11 @@ export function bucketGrants(bucket: string, profiles: readonly string[]): Condi
|
|
|
51
51
|
`resource.name.startsWith("projects/_/buckets/${bucket}/objects/${path}")`;
|
|
52
52
|
const objectIs = (path: string): string =>
|
|
53
53
|
`resource.name == "projects/_/buckets/${bucket}/objects/${path}"`;
|
|
54
|
+
// Parenthesised, because `!` binds tighter than `==` in CEL: `!resource.name
|
|
55
|
+
// == "…"` negates the *name* and compares that, which is not a type error and
|
|
56
|
+
// not what it reads as. Bracketing is the whole difference between excluding
|
|
57
|
+
// one object and excluding nothing.
|
|
58
|
+
const objectIsNot = (path: string): string => `!(${objectIs(path)})`;
|
|
54
59
|
/**
|
|
55
60
|
* The bucket itself, which is a different resource from anything in it.
|
|
56
61
|
*
|
|
@@ -80,10 +85,41 @@ export function bucketGrants(bucket: string, profiles: readonly string[]): Condi
|
|
|
80
85
|
// One prefix, not one per profile. Manifests are the workspace's since
|
|
81
86
|
// ADR-057 — a manifest defines a connection, and connections do not live in a
|
|
82
87
|
// profile — so the carve-out no longer varies with the profile set.
|
|
83
|
-
const
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
88
|
+
const manifests = `(${objectsUnder(`${layout.providers()}/`)})`;
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* What a running revision writes, named rather than carved out of `data/`.
|
|
92
|
+
*
|
|
93
|
+
* It used to be `objectsUnder('data/') && !manifests` — everything under one
|
|
94
|
+
* prefix, minus the exception. There is no `data/` any more (ADR-067), and
|
|
95
|
+
* naming the writable prefixes is the better shape regardless: an allowlist
|
|
96
|
+
* says what a compromised revision can reach, where a denylist says only what
|
|
97
|
+
* it cannot and grows silently every time something new lands under the
|
|
98
|
+
* prefix.
|
|
99
|
+
*
|
|
100
|
+
* `credentials.enc` is in it because `connect --from-endpoint` and a token
|
|
101
|
+
* refresh both write there (ADR-025), and its `.key` sits beside it.
|
|
102
|
+
*/
|
|
103
|
+
// **Per profile, not `profiles/` wholesale.** Granting the whole tree and
|
|
104
|
+
// carving the served declarations back out left the revision `create` and
|
|
105
|
+
// `delete` on every *other* profile's `profile.yaml` — the configuration of a
|
|
106
|
+
// profile the same endpoint serves — and on any name it invented, which
|
|
107
|
+
// `listProfiles` would then pick up on the next boot. Before ADR-067 no
|
|
108
|
+
// declaration was writable at all, because `profiles/` sat outside `data/`;
|
|
109
|
+
// this restores that, one prefix per profile.
|
|
110
|
+
//
|
|
111
|
+
// `!==` on a whole object rather than a pattern, because Cloud Storage IAM
|
|
112
|
+
// conditions have no `matches` — see the note on the read binding below.
|
|
113
|
+
const writable = [
|
|
114
|
+
objectsUnder(`${layout.audit()}/`),
|
|
115
|
+
objectsUnder(`${layout.state()}/`),
|
|
116
|
+
objectsUnder(layout.credentials()),
|
|
117
|
+
...profiles.map(
|
|
118
|
+
(profile) =>
|
|
119
|
+
`(${objectsUnder(`${layout.profileDir(profile)}/`)} && ` +
|
|
120
|
+
`${objectIsNot(layout.profileConfig(profile))})`,
|
|
121
|
+
),
|
|
122
|
+
].join(' || ');
|
|
87
123
|
|
|
88
124
|
return [
|
|
89
125
|
{
|
|
@@ -91,7 +127,7 @@ export function bucketGrants(bucket: string, profiles: readonly string[]): Condi
|
|
|
91
127
|
// skills are all written by the running endpoint.
|
|
92
128
|
role: 'roles/storage.objectAdmin',
|
|
93
129
|
title: 'owns-its-data',
|
|
94
|
-
expression:
|
|
130
|
+
expression: `(${writable})`,
|
|
95
131
|
},
|
|
96
132
|
{
|
|
97
133
|
// `expression=true` was here, which is every object in the bucket — the
|
|
@@ -110,7 +146,7 @@ export function bucketGrants(bucket: string, profiles: readonly string[]): Condi
|
|
|
110
146
|
// saw a passing build and Cloud Run saw a container that never started.
|
|
111
147
|
role: 'roles/storage.objectViewer',
|
|
112
148
|
title: 'reads-its-config',
|
|
113
|
-
expression: `${theBucket} || ${objectsUnder(
|
|
149
|
+
expression: `${theBucket} || ${objectsUnder(`${layout.profilesRoot()}/`)} || ${objectIs(WORKSPACE_FILE)} || ${objectIs(CONNECTIONS_FILE)} || ${manifests}`,
|
|
114
150
|
},
|
|
115
151
|
];
|
|
116
152
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { SecretStore } from '#secrets';
|
|
2
2
|
import type { BlobStore } from '#stores/blobs';
|
|
3
3
|
import type { BlobRoute } from '#stores/blobs/route.ts';
|
|
4
|
-
import { KNOWLEDGE_LAYOUT, knowledgeRoot, type KnowledgeArea, type KnowledgeConfig } from '#profile';
|
|
4
|
+
import { KNOWLEDGE_LAYOUT, KNOWLEDGE_PREFIX, knowledgeRoot, type KnowledgeArea, type KnowledgeConfig } from '#profile';
|
|
5
5
|
import { requireSecret, type TargetInput } from './target.ts';
|
|
6
6
|
import type { FetchLike } from './adapters/github-api.ts';
|
|
7
7
|
// Type-only, so a target with no `knowledge` block never loads the adapter.
|
|
@@ -134,10 +134,15 @@ export async function knowledgeStores(
|
|
|
134
134
|
* how they would come to disagree. `skills` is absent because it is not a
|
|
135
135
|
* prefix of that root — it is a store of its own, handed over whole.
|
|
136
136
|
*/
|
|
137
|
-
export function knowledgeRoutes(
|
|
137
|
+
export function knowledgeRoutes(
|
|
138
|
+
stores: Pick<KnowledgeStores, 'memory' | 'entities'>,
|
|
139
|
+
): BlobRoute[] {
|
|
138
140
|
return [
|
|
139
|
-
|
|
140
|
-
|
|
141
|
+
// The *provider's* prefix, which is what core scopes a store into. The
|
|
142
|
+
// repository directory it lands in is `KNOWLEDGE_LAYOUT`, and the two are
|
|
143
|
+
// different strings since contract 4 prefixed the owner layer.
|
|
144
|
+
{ prefix: `${KNOWLEDGE_PREFIX.memory}/`, store: stores.memory },
|
|
145
|
+
{ prefix: `${KNOWLEDGE_PREFIX.entities}/`, store: stores.entities },
|
|
141
146
|
];
|
|
142
147
|
}
|
|
143
148
|
|