@nylorun/runtime 0.19.1-beta → 0.21.0-beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +90 -0
- package/README.md +22 -13
- package/dist/adapters/streams/s2.js +5 -11
- package/dist/api/components.d.ts +326 -205
- package/dist/api/components.js +3 -3
- package/dist/api/http/app.js +2 -0
- package/dist/api/http/define.d.ts +3 -2
- package/dist/api/http/define.js +10 -11
- package/dist/api/http/routes/mcp.d.ts +9 -0
- package/dist/api/http/routes/mcp.js +32 -0
- package/dist/api/http/routes/vaults.d.ts +1 -9
- package/dist/api/http/routes/vaults.js +3 -103
- package/dist/api/openapi.js +37 -6
- package/dist/artifacts/media-types.d.ts +2 -0
- package/dist/artifacts/media-types.js +5 -0
- package/dist/execution/types.d.ts +1 -1
- package/dist/gates/http-tool.d.ts +7 -0
- package/dist/gates/http-tool.js +34 -3
- package/dist/gates/mcp-handler.d.ts +4 -1
- package/dist/gates/mcp-handler.js +64 -8
- package/dist/gates/tenant-vaults.d.ts +9 -15
- package/dist/gates/tenant-vaults.js +9 -17
- package/dist/gates/tool-client.js +36 -17
- package/dist/gates/tool-contract.d.ts +19 -3
- package/dist/gates/tool-contract.js +9 -3
- package/dist/gates/tool-gate.d.ts +2 -2
- package/dist/harness/calls.d.ts +2 -0
- package/dist/harness/calls.js +13 -3
- package/dist/harness-api/record.d.ts +7 -6
- package/dist/harness-api/record.js +99 -17
- package/dist/harness-api/server.js +35 -27
- package/dist/harness-api/start.js +2 -1
- package/dist/host/app.d.ts +3 -0
- package/dist/host/app.js +33 -4
- package/dist/host/config-for.d.ts +0 -2
- package/dist/host/config-for.js +0 -1
- package/dist/host/create-host.d.ts +6 -0
- package/dist/host/create-host.js +1 -0
- package/dist/host/gates.js +0 -1
- package/dist/host/main.js +24 -7
- package/dist/host/stack-config.d.ts +5 -8
- package/dist/host/stack-config.js +9 -20
- package/dist/index.d.ts +0 -1
- package/dist/infra/index.d.ts +2 -1
- package/dist/infra/index.js +0 -1
- package/dist/infra/readiness.d.ts +6 -6
- package/dist/infra/readiness.js +2 -5
- package/dist/keys/client.js +1 -9
- package/dist/keys/keys.d.ts +11 -13
- package/dist/keys/keys.js +5 -7
- package/dist/management-openapi.json +671 -378
- package/dist/mcp/connect.d.ts +159 -13
- package/dist/mcp/connect.js +538 -55
- package/dist/mcp/pool.d.ts +11 -2
- package/dist/mcp/pool.js +74 -27
- package/dist/mcp/preview.d.ts +35 -0
- package/dist/mcp/preview.js +215 -0
- package/dist/mcp/snapshot.d.ts +79 -5
- package/dist/mcp/snapshot.js +145 -27
- package/dist/model/catalog.d.ts +14 -0
- package/dist/model/catalog.js +14 -1
- package/dist/model/pi-model.js +36 -2
- package/dist/openapi.json +644 -266
- package/dist/redact.d.ts +19 -1
- package/dist/redact.js +42 -13
- package/dist/sandbox/session-sandbox.d.ts +7 -0
- package/dist/sandbox/session-sandbox.js +89 -28
- package/dist/store/postgres/connect.d.ts +10 -0
- package/dist/store/postgres/connect.js +30 -3
- package/dist/store/postgres/control.d.ts +14 -0
- package/dist/store/postgres/control.js +187 -0
- package/dist/store/postgres/drizzle/0015_control_signals.sql +11 -0
- package/dist/store/postgres/drizzle/0016_mcp_oauth_removed.sql +5 -0
- package/dist/store/postgres/drizzle/meta/_journal.json +14 -0
- package/dist/store/postgres/schema.d.ts +122 -261
- package/dist/store/postgres/schema.js +22 -36
- package/dist/store/postgres/store.js +16 -18
- package/dist/store/postgres/tenant.d.ts +14 -0
- package/dist/store/postgres/tenant.js +22 -2
- package/dist/store/types.d.ts +42 -8
- package/dist/streams/types.d.ts +6 -29
- package/dist/streams/types.js +5 -4
- package/dist/tenant/artifact-tool.d.ts +9 -1
- package/dist/tenant/artifact-tool.js +113 -9
- package/dist/tenant/auth.d.ts +16 -6
- package/dist/tenant/auth.js +41 -17
- package/dist/tenant/commands.js +10 -6
- package/dist/tenant/context.d.ts +9 -6
- package/dist/tenant/control.d.ts +26 -0
- package/dist/tenant/control.js +35 -0
- package/dist/tenant/effects.d.ts +2 -2
- package/dist/tenant/effects.js +20 -5
- package/dist/tenant/ephemeral.d.ts +0 -6
- package/dist/tenant/ephemeral.js +1 -1
- package/dist/tenant/http.d.ts +4 -2
- package/dist/tenant/http.js +4 -2
- package/dist/tenant/identity-file.d.ts +16 -12
- package/dist/tenant/identity-file.js +41 -29
- package/dist/tenant/issuers.d.ts +4 -3
- package/dist/tenant/issuers.js +22 -16
- package/dist/tenant/outbound.d.ts +14 -5
- package/dist/tenant/outbound.js +21 -9
- package/dist/tenant/reset.d.ts +2 -2
- package/dist/tenant/reset.js +8 -2
- package/dist/tenant/resource-server.d.ts +56 -0
- package/dist/tenant/resource-server.js +62 -0
- package/dist/tenant/runtime.d.ts +1 -1
- package/dist/tenant/runtime.js +16 -15
- package/dist/tenant/scheduler.d.ts +1 -1
- package/dist/tenant/scheduler.js +1 -1
- package/dist/tenant/sessions.js +20 -3
- package/dist/tenant/store-pg.js +9 -0
- package/dist/tenant/streams.d.ts +16 -22
- package/dist/tenant/streams.js +7 -87
- package/dist/tenant/sweep.js +9 -1
- package/dist/tenant/tool-results.d.ts +57 -0
- package/dist/tenant/tool-results.js +298 -0
- package/dist/tenant/tool-search.d.ts +43 -0
- package/dist/tenant/tool-search.js +100 -0
- package/dist/tenant/transcript.d.ts +108 -0
- package/dist/tenant/transcript.js +13 -0
- package/dist/tenant/types.d.ts +0 -14
- package/dist/vault/crypto.d.ts +0 -7
- package/dist/vault/crypto.js +0 -16
- package/dist/vault/headers.d.ts +17 -0
- package/dist/vault/headers.js +61 -0
- package/dist/vault/service.d.ts +11 -68
- package/dist/vault/service.js +83 -438
- package/dist/vault/sources.d.ts +20 -48
- package/dist/vault/sources.js +25 -189
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +3 -3
- package/dist/streams/control.d.ts +0 -13
- package/dist/streams/control.js +0 -25
- package/dist/vault/oauth.d.ts +0 -55
- package/dist/vault/oauth.js +0 -186
package/dist/redact.d.ts
CHANGED
|
@@ -1 +1,19 @@
|
|
|
1
|
-
export
|
|
1
|
+
export interface ScrubOptions {
|
|
2
|
+
/**
|
|
3
|
+
* Replace only `secrets` (R2b C8, Q16): no value under a secret-looking key, and no inline
|
|
4
|
+
* image data. A tool's answer keeps fields such as `nextToken`.
|
|
5
|
+
*/
|
|
6
|
+
readonly valuesOnly?: boolean;
|
|
7
|
+
/** Told how many times a secret was replaced. */
|
|
8
|
+
readonly counted?: (replacements: number) => void;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* `value` with each of `secrets` (those of at least 8 characters) replaced by `[redacted]`. By
|
|
12
|
+
* default also any value under a key such as `token` or `secret`, and inline image data.
|
|
13
|
+
*/
|
|
14
|
+
export declare function scrub(value: unknown, secrets: readonly string[], options?: ScrubOptions): unknown;
|
|
15
|
+
/** `value` with `secrets` alone replaced (`valuesOnly`), and how many replacements were made. */
|
|
16
|
+
export declare function scrubValues<T>(value: T, secrets: readonly string[]): {
|
|
17
|
+
value: T;
|
|
18
|
+
redacted: number;
|
|
19
|
+
};
|
package/dist/redact.js
CHANGED
|
@@ -1,14 +1,43 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
1
|
+
/**
|
|
2
|
+
* `value` with each of `secrets` (those of at least 8 characters) replaced by `[redacted]`. By
|
|
3
|
+
* default also any value under a key such as `token` or `secret`, and inline image data.
|
|
4
|
+
*/
|
|
5
|
+
export function scrub(value, secrets, options = {}) {
|
|
6
|
+
// The longest first, so a header value goes whole before the token inside it.
|
|
7
|
+
const sorted = secrets.filter((secret) => secret.length >= 8).sort((a, b) => b.length - a.length);
|
|
8
|
+
let replaced = 0;
|
|
9
|
+
const walk = (item) => {
|
|
10
|
+
if (typeof item === "string") {
|
|
11
|
+
const text = options.valuesOnly
|
|
12
|
+
? item
|
|
13
|
+
: item.replace(/data:image\/[a-z0-9.+-]+;base64,[A-Za-z0-9+/=]+/giu, "[inline image data redacted]");
|
|
14
|
+
return sorted.reduce((current, secret) => {
|
|
15
|
+
const parts = current.split(secret);
|
|
16
|
+
replaced += parts.length - 1;
|
|
17
|
+
return parts.join("[redacted]");
|
|
18
|
+
}, text);
|
|
19
|
+
}
|
|
20
|
+
if (Array.isArray(item))
|
|
21
|
+
return item.map(walk);
|
|
22
|
+
if (item && typeof item === "object")
|
|
23
|
+
return Object.fromEntries(Object.entries(item).map(([key, entry]) => [
|
|
24
|
+
key,
|
|
25
|
+
!options.valuesOnly &&
|
|
26
|
+
/^(authorization|api[_-]?key|(?:access|refresh|id|auth)[_-]?token|token|secret|password|cookie|credentials?)$/iu.test(key)
|
|
27
|
+
? "[redacted]"
|
|
28
|
+
: walk(entry),
|
|
29
|
+
]));
|
|
30
|
+
return item;
|
|
31
|
+
};
|
|
32
|
+
const result = walk(value);
|
|
33
|
+
options.counted?.(replaced);
|
|
34
|
+
return result;
|
|
35
|
+
}
|
|
36
|
+
/** `value` with `secrets` alone replaced (`valuesOnly`), and how many replacements were made. */
|
|
37
|
+
export function scrubValues(value, secrets) {
|
|
38
|
+
if (secrets.length === 0)
|
|
39
|
+
return { value, redacted: 0 };
|
|
40
|
+
let redacted = 0;
|
|
41
|
+
const scrubbed = scrub(value, secrets, { valuesOnly: true, counted: (count) => (redacted = count) });
|
|
42
|
+
return { value: redacted === 0 ? value : scrubbed, redacted };
|
|
14
43
|
}
|
|
@@ -9,3 +9,10 @@ export type SandboxedManifest = {
|
|
|
9
9
|
};
|
|
10
10
|
/** The agent's manifest with the sandbox capability added, or why it cannot take one. */
|
|
11
11
|
export declare function withSandboxCapability(manifest: AgentManifest, spec: SandboxManifest): SandboxedManifest;
|
|
12
|
+
/**
|
|
13
|
+
* The pinned manifest of an agent session without a sandbox: `read_artifact` for each agent of
|
|
14
|
+
* the tree with a remote MCP server or an HTTP tool (R2b C11), and `nylorun.tools` for each with
|
|
15
|
+
* a remote MCP server (R2b C10). Undefined when nothing is added, and the session pins its
|
|
16
|
+
* definition as it is.
|
|
17
|
+
*/
|
|
18
|
+
export declare function withPlatformTools(manifest: AgentManifest): SandboxedManifest | undefined;
|
|
@@ -1,13 +1,23 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Gives a session the sandbox chosen when it was opened
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
2
|
+
* Gives a session the sandbox chosen when it was opened, and the platform tools its agents
|
|
3
|
+
* need. An agent session's pinned manifest is its definition plus:
|
|
4
|
+
*
|
|
5
|
+
* - with a sandbox, the `nylorun.sandbox` capability (the six built-in tools, the sandbox
|
|
6
|
+
* instructions and the resolved spec) and the `nylorun.artifacts` capability
|
|
7
|
+
* (`save_artifact`), also added to each agent it uses as a tool, so the tree shares one spec.
|
|
8
|
+
* The sandbox's instructions name the agent's skills' folders (`/skills/<name>/`);
|
|
9
|
+
* - for each agent of the tree with a remote MCP server or an HTTP tool, `read_artifact` in its
|
|
10
|
+
* `nylorun.artifacts` capability (R2b C11), with or without a sandbox: the Runtime stores a
|
|
11
|
+
* result of those tools too large to show as an artifact, and the model reads it in pages.
|
|
12
|
+
* An agent that declares a tool named `read_artifact` keeps its own, and sees previews only;
|
|
13
|
+
* - for each agent of the tree with a remote MCP server, the `nylorun.tools` capability (R2b
|
|
14
|
+
* C10), empty: when the session defers some of the agent's MCP tools, its session tools put
|
|
15
|
+
* `tool_search` and `tool_call` there. An agent that declares either name defers nothing.
|
|
16
|
+
*
|
|
17
|
+
* The definition in the registry never changes.
|
|
8
18
|
*/
|
|
9
19
|
import { AgentManifestSchema } from "@nylorun/core/contracts";
|
|
10
|
-
import { artifactsCapabilityManifest, hashManifest, sandboxCapabilityManifest, } from "@nylorun/core/define";
|
|
20
|
+
import { READ_ARTIFACT_TOOL, TOOL_CALL_TOOL, TOOL_SEARCH_TOOL, TOOLS_CAPABILITY_ID, artifactsCapabilityManifest, hashManifest, sandboxCapabilityManifest, toolsCapabilityManifest, } from "@nylorun/core/define";
|
|
11
21
|
import { sandboxSpecOf } from "./share.js";
|
|
12
22
|
function isAgentManifest(value) {
|
|
13
23
|
return (typeof value === "object" &&
|
|
@@ -19,35 +29,72 @@ function isAgentManifest(value) {
|
|
|
19
29
|
function skillNamesOf(manifest) {
|
|
20
30
|
return manifest.capabilities.flatMap((capability) => Object.keys(capability.skills ?? {})).sort();
|
|
21
31
|
}
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
32
|
+
/**
|
|
33
|
+
* True when the agent itself declares a remote MCP server or an HTTP tool, whose results the
|
|
34
|
+
* Runtime may store as artifacts (R2b C11), and no tool of its own is named `read_artifact`.
|
|
35
|
+
*/
|
|
36
|
+
function readsStoredResults(manifest) {
|
|
37
|
+
const remote = manifest.capabilities.some((capability) => Object.keys(capability.mcpServers ?? {}).length > 0 ||
|
|
38
|
+
(capability.tools ?? []).some((tool) => tool.http !== undefined));
|
|
39
|
+
const taken = manifest.capabilities.some((capability) => (capability.tools ?? []).some((tool) => tool.name === READ_ARTIFACT_TOOL));
|
|
40
|
+
return remote && !taken;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* True when the agent itself declares a remote MCP server, some of whose tools the session may
|
|
44
|
+
* defer (R2b C10), and names no tool `tool_search` or `tool_call` (it then keeps every tool in
|
|
45
|
+
* its list).
|
|
46
|
+
*/
|
|
47
|
+
function defersTools(manifest) {
|
|
48
|
+
const servers = manifest.capabilities.some((capability) => Object.keys(capability.mcpServers ?? {}).length > 0);
|
|
49
|
+
const taken = manifest.capabilities.some((capability) => capability.id === TOOLS_CAPABILITY_ID ||
|
|
50
|
+
(capability.tools ?? []).some((tool) => tool.name === TOOL_SEARCH_TOOL || tool.name === TOOL_CALL_TOOL));
|
|
51
|
+
return servers && !taken;
|
|
52
|
+
}
|
|
53
|
+
/** `manifest` with the session's capabilities added; the same object when none are. */
|
|
54
|
+
function addCapabilities(manifest, spec) {
|
|
55
|
+
let changed = false;
|
|
56
|
+
const capabilities = manifest.capabilities.map((capability) => {
|
|
57
|
+
if (capability.tools === undefined)
|
|
58
|
+
return capability;
|
|
59
|
+
let toolsChanged = false;
|
|
60
|
+
const tools = capability.tools.map((tool) => {
|
|
61
|
+
// Agents used as tools share the session's sandbox; flow agents used as tools run in
|
|
62
|
+
// linked sessions that inherit it. One that declares its own sandbox keeps it.
|
|
63
|
+
if (tool.agent === undefined || !isAgentManifest(tool.agent))
|
|
64
|
+
return tool;
|
|
65
|
+
const agent = addCapabilities(tool.agent, sandboxSpecOf(tool.agent) === undefined ? spec : undefined);
|
|
66
|
+
if (agent === tool.agent)
|
|
67
|
+
return tool;
|
|
68
|
+
toolsChanged = true;
|
|
69
|
+
return { ...tool, agent };
|
|
36
70
|
});
|
|
71
|
+
if (!toolsChanged)
|
|
72
|
+
return capability;
|
|
73
|
+
changed = true;
|
|
74
|
+
return { ...capability, tools };
|
|
75
|
+
});
|
|
76
|
+
const save = spec !== undefined;
|
|
77
|
+
const read = readsStoredResults(manifest);
|
|
78
|
+
const defer = defersTools(manifest);
|
|
79
|
+
if (!save && !read && !defer && !changed)
|
|
80
|
+
return manifest;
|
|
37
81
|
return {
|
|
38
82
|
...manifest,
|
|
39
83
|
// With a sandbox comes `save_artifact`, so files the agent makes reach the user (F8.1). The
|
|
40
84
|
// sandbox's instructions name where the agent's skills are (R2 M4).
|
|
41
85
|
capabilities: [
|
|
42
86
|
...capabilities,
|
|
43
|
-
|
|
44
|
-
|
|
87
|
+
...(spec === undefined
|
|
88
|
+
? []
|
|
89
|
+
: [sandboxCapabilityManifest(spec, { skills: skillNamesOf(manifest) })]),
|
|
90
|
+
...(save || read ? [artifactsCapabilityManifest({ save, read })] : []),
|
|
91
|
+
// Empty: when the session defers MCP tools, its tools put tool_search and tool_call here.
|
|
92
|
+
...(defer ? [toolsCapabilityManifest()] : []),
|
|
45
93
|
],
|
|
46
94
|
};
|
|
47
95
|
}
|
|
48
|
-
/**
|
|
49
|
-
|
|
50
|
-
const sandboxed = addCapability(manifest, spec);
|
|
96
|
+
/** `sandboxed` checked as a manifest, with its hash. */
|
|
97
|
+
function checked(original, sandboxed) {
|
|
51
98
|
const parsed = AgentManifestSchema.safeParse(sandboxed);
|
|
52
99
|
if (!parsed.success) {
|
|
53
100
|
const issues = parsed.error.issues.map((issue) => issue.message);
|
|
@@ -55,9 +102,23 @@ export function withSandboxCapability(manifest, spec) {
|
|
|
55
102
|
return {
|
|
56
103
|
ok: false,
|
|
57
104
|
message: clash
|
|
58
|
-
? `'${
|
|
59
|
-
: `'${
|
|
105
|
+
? `'${original.id}' declares a tool named like a sandbox tool (bash, read, write, edit, grep, glob, save_artifact). Rename it, or open the session with sandbox: false.`
|
|
106
|
+
: `'${original.id}' cannot take this sandbox: ${[...new Set(issues)].join("; ")}`,
|
|
60
107
|
};
|
|
61
108
|
}
|
|
62
109
|
return { ok: true, manifest: sandboxed, manifestHash: hashManifest(sandboxed) };
|
|
63
110
|
}
|
|
111
|
+
/** The agent's manifest with the sandbox capability added, or why it cannot take one. */
|
|
112
|
+
export function withSandboxCapability(manifest, spec) {
|
|
113
|
+
return checked(manifest, addCapabilities(manifest, spec));
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* The pinned manifest of an agent session without a sandbox: `read_artifact` for each agent of
|
|
117
|
+
* the tree with a remote MCP server or an HTTP tool (R2b C11), and `nylorun.tools` for each with
|
|
118
|
+
* a remote MCP server (R2b C10). Undefined when nothing is added, and the session pins its
|
|
119
|
+
* definition as it is.
|
|
120
|
+
*/
|
|
121
|
+
export function withPlatformTools(manifest) {
|
|
122
|
+
const pinned = addCapabilities(manifest, undefined);
|
|
123
|
+
return pinned === manifest ? undefined : checked(manifest, pinned);
|
|
124
|
+
}
|
|
@@ -41,3 +41,13 @@ export declare function createPostgresClient(url: string, options?: PostgresClie
|
|
|
41
41
|
export declare function assertLogicalReplication(sql: Sql): Promise<void>;
|
|
42
42
|
/** Separate, bounded pool for read projections; inherits the Tenant connection credentials. */
|
|
43
43
|
export declare function createPostgresReadClient(source: PostgresClient): PostgresClient;
|
|
44
|
+
/**
|
|
45
|
+
* One connection of its own for `LISTEN` (the control bus, `control.ts`), with `source`'s
|
|
46
|
+
* database and credentials. It never idles out. `onClose` runs when the connection ends for
|
|
47
|
+
* any reason; the caller listens again, which connects again. A `LISTEN` needs a session of
|
|
48
|
+
* its own: behind a pooler in transaction mode it would not hear a thing.
|
|
49
|
+
*/
|
|
50
|
+
export declare function createPostgresListenClient(source: PostgresClient, handlers: {
|
|
51
|
+
onNotify: (channel: string, payload: string) => void;
|
|
52
|
+
onClose: () => void;
|
|
53
|
+
}): PostgresClient;
|
|
@@ -40,9 +40,9 @@ export async function assertLogicalReplication(sql) {
|
|
|
40
40
|
if (!row.replicates)
|
|
41
41
|
throw new Error("The Runtime's Postgres role cannot replicate: grant it REPLICATION (ALTER ROLE … REPLICATION). See DEPLOYMENT.md.");
|
|
42
42
|
}
|
|
43
|
-
/**
|
|
44
|
-
|
|
45
|
-
|
|
43
|
+
/** Where `source` connects, as whom: for clients that share its database and credentials. */
|
|
44
|
+
function connectionOf(source) {
|
|
45
|
+
return {
|
|
46
46
|
host: source.options.host.length === 1
|
|
47
47
|
? source.options.host[0]
|
|
48
48
|
: source.options.host
|
|
@@ -54,6 +54,12 @@ export function createPostgresReadClient(source) {
|
|
|
54
54
|
pass: source.options.pass ?? "",
|
|
55
55
|
ssl: source.options.ssl,
|
|
56
56
|
path: source.options.path,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
/** Separate, bounded pool for read projections; inherits the Tenant connection credentials. */
|
|
60
|
+
export function createPostgresReadClient(source) {
|
|
61
|
+
const options = {
|
|
62
|
+
...connectionOf(source),
|
|
57
63
|
max: 4,
|
|
58
64
|
idle_timeout: 30,
|
|
59
65
|
connect_timeout: 2,
|
|
@@ -69,3 +75,24 @@ export function createPostgresReadClient(source) {
|
|
|
69
75
|
const pool = postgres(options);
|
|
70
76
|
return pool;
|
|
71
77
|
}
|
|
78
|
+
/**
|
|
79
|
+
* One connection of its own for `LISTEN` (the control bus, `control.ts`), with `source`'s
|
|
80
|
+
* database and credentials. It never idles out. `onClose` runs when the connection ends for
|
|
81
|
+
* any reason; the caller listens again, which connects again. A `LISTEN` needs a session of
|
|
82
|
+
* its own: behind a pooler in transaction mode it would not hear a thing.
|
|
83
|
+
*/
|
|
84
|
+
export function createPostgresListenClient(source, handlers) {
|
|
85
|
+
const options = {
|
|
86
|
+
...connectionOf(source),
|
|
87
|
+
max: 1,
|
|
88
|
+
idle_timeout: 0,
|
|
89
|
+
max_lifetime: null,
|
|
90
|
+
connect_timeout: 10,
|
|
91
|
+
fetch_types: false,
|
|
92
|
+
onnotice() { },
|
|
93
|
+
onnotify: handlers.onNotify,
|
|
94
|
+
onclose: () => handlers.onClose(),
|
|
95
|
+
connection: { application_name: "nylorun-control" },
|
|
96
|
+
};
|
|
97
|
+
return postgres(options);
|
|
98
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { Sql } from "postgres";
|
|
2
|
+
import type { ControlSignal, FollowSignalsOptions, SignalFollower } from "../types.js";
|
|
3
|
+
import type { Database, Queryable } from "./db.js";
|
|
4
|
+
/** The channel every signal is notified on. */
|
|
5
|
+
export declare const CONTROL_CHANNEL = "nylorun_control";
|
|
6
|
+
/** Puts `signal` on the bus in the transaction `db` runs: a row, notified at commit. */
|
|
7
|
+
export declare function writeSignal(db: Queryable, signal: ControlSignal): Promise<void>;
|
|
8
|
+
/** Deletes the signals written before `before`; returns how many. */
|
|
9
|
+
export declare function pruneSignals(db: Queryable, before: Date): Promise<number>;
|
|
10
|
+
/**
|
|
11
|
+
* Follows the bus on `source`'s database (`SessionStore.followSignals`). Resolves once the
|
|
12
|
+
* start is fixed; the first LISTEN may still be retrying, and the poll covers it meanwhile.
|
|
13
|
+
*/
|
|
14
|
+
export declare function followControlSignals(source: Sql, db: Database, onSignal: (signal: ControlSignal) => void, options?: FollowSignalsOptions): Promise<SignalFollower>;
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The control bus on Postgres (blueprint D21, D48): how one process tells the others with the
|
|
3
|
+
* Tenant open to act now. `session.cancel` aborts the advance of a cancelled turn;
|
|
4
|
+
* `sessions.reset` moves the session streams to the Tenant's new basin generation. Nothing
|
|
5
|
+
* internal goes through S2, which only serves API listeners.
|
|
6
|
+
*
|
|
7
|
+
* - **Write.** `writeSignal` inserts a `control_signals` row and calls `pg_notify` on
|
|
8
|
+
* `nylorun_control`, in the caller's transaction. Postgres sends the notification to every
|
|
9
|
+
* listening session when the transaction commits and drops it on rollback, so a signal
|
|
10
|
+
* exists exactly when what it announces committed.
|
|
11
|
+
* - **Follow.** `followControlSignals` holds one connection of its own that LISTENs (a pooled
|
|
12
|
+
* connection cannot). Each notification reads back the signals of the last
|
|
13
|
+
* `WINDOW_SECONDS` and delivers those this follower has not delivered yet. So does a poll
|
|
14
|
+
* every `pollMs`, and every LISTEN after a reconnect: Postgres does not queue notifications
|
|
15
|
+
* for a session that is not connected, and the rows are what catches up. Ids are never read
|
|
16
|
+
* as a cursor: they are allocated at insert and commit out of order, so a later id can be
|
|
17
|
+
* visible first.
|
|
18
|
+
* - **Start.** A follower delivers the signals written from its start on (the database's
|
|
19
|
+
* clock), never older ones.
|
|
20
|
+
* - **Prune.** A row outlives its notification only for the read-back; the Tenant sweep
|
|
21
|
+
* deletes old ones (`pruneSignals`).
|
|
22
|
+
*/
|
|
23
|
+
import { gte, lt, sql } from "drizzle-orm";
|
|
24
|
+
import { createPostgresListenClient } from "./connect.js";
|
|
25
|
+
import { controlSignals } from "./schema.js";
|
|
26
|
+
/** The channel every signal is notified on. */
|
|
27
|
+
export const CONTROL_CHANNEL = "nylorun_control";
|
|
28
|
+
/** How far back a read-back looks. Longer than the poll, so a missed notification is caught. */
|
|
29
|
+
const WINDOW_SECONDS = 120;
|
|
30
|
+
const POLL_MS = 5000;
|
|
31
|
+
const RETRY_MIN_MS = 100;
|
|
32
|
+
const RETRY_MAX_MS = 5000;
|
|
33
|
+
const SIGNAL = {
|
|
34
|
+
id: controlSignals.id,
|
|
35
|
+
kind: controlSignals.kind,
|
|
36
|
+
sessionId: controlSignals.sessionId,
|
|
37
|
+
turnId: controlSignals.turnId,
|
|
38
|
+
generation: controlSignals.generation,
|
|
39
|
+
};
|
|
40
|
+
/** Puts `signal` on the bus in the transaction `db` runs: a row, notified at commit. */
|
|
41
|
+
export async function writeSignal(db, signal) {
|
|
42
|
+
const [row] = await db
|
|
43
|
+
.insert(controlSignals)
|
|
44
|
+
.values({
|
|
45
|
+
kind: signal.type,
|
|
46
|
+
sessionId: signal.type === "session.cancel" ? signal.sessionId : null,
|
|
47
|
+
turnId: signal.type === "session.cancel" ? (signal.turnId ?? null) : null,
|
|
48
|
+
generation: signal.type === "sessions.reset" ? signal.generation : null,
|
|
49
|
+
})
|
|
50
|
+
.returning({ id: controlSignals.id });
|
|
51
|
+
// The payload is only the id: a follower reads the rows back either way.
|
|
52
|
+
await db.execute(sql `SELECT pg_notify(${CONTROL_CHANNEL}, ${String(row.id)})`);
|
|
53
|
+
}
|
|
54
|
+
/** Deletes the signals written before `before`; returns how many. */
|
|
55
|
+
export async function pruneSignals(db, before) {
|
|
56
|
+
const deleted = await db
|
|
57
|
+
.delete(controlSignals)
|
|
58
|
+
.where(lt(controlSignals.createdAt, before.toISOString()))
|
|
59
|
+
.returning({ id: controlSignals.id });
|
|
60
|
+
return deleted.length;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Follows the bus on `source`'s database (`SessionStore.followSignals`). Resolves once the
|
|
64
|
+
* start is fixed; the first LISTEN may still be retrying, and the poll covers it meanwhile.
|
|
65
|
+
*/
|
|
66
|
+
export async function followControlSignals(source, db, onSignal, options = {}) {
|
|
67
|
+
const pollMs = options.pollMs ?? POLL_MS;
|
|
68
|
+
const onError = options.onError ?? (() => { });
|
|
69
|
+
// Drizzle leaves timestamps to its column types, so a raw read of one is text.
|
|
70
|
+
const [start] = await db.execute(sql `SELECT clock_timestamp()::text AS now`);
|
|
71
|
+
const since = start.now;
|
|
72
|
+
/** Ids delivered, with when (this process's clock), until no read-back can return them. */
|
|
73
|
+
const delivered = new Map();
|
|
74
|
+
let stopped = false;
|
|
75
|
+
let retry;
|
|
76
|
+
let delay = RETRY_MIN_MS;
|
|
77
|
+
let draining;
|
|
78
|
+
let again = false;
|
|
79
|
+
const client = createPostgresListenClient(source, {
|
|
80
|
+
onNotify: (channel) => {
|
|
81
|
+
if (channel === CONTROL_CHANNEL)
|
|
82
|
+
drain();
|
|
83
|
+
},
|
|
84
|
+
// The connection ended (Postgres restarted, the network dropped): listen again, which
|
|
85
|
+
// connects again, and read back what was missed meanwhile.
|
|
86
|
+
onClose: () => relisten(),
|
|
87
|
+
});
|
|
88
|
+
function relisten() {
|
|
89
|
+
if (stopped || retry)
|
|
90
|
+
return;
|
|
91
|
+
retry = setTimeout(() => {
|
|
92
|
+
retry = undefined;
|
|
93
|
+
void listen();
|
|
94
|
+
}, delay);
|
|
95
|
+
retry.unref();
|
|
96
|
+
delay = Math.min(delay * 2, RETRY_MAX_MS);
|
|
97
|
+
}
|
|
98
|
+
async function listen() {
|
|
99
|
+
if (stopped)
|
|
100
|
+
return;
|
|
101
|
+
try {
|
|
102
|
+
await client.unsafe(`LISTEN ${CONTROL_CHANNEL}`);
|
|
103
|
+
delay = RETRY_MIN_MS;
|
|
104
|
+
drain();
|
|
105
|
+
}
|
|
106
|
+
catch (error) {
|
|
107
|
+
if (stopped)
|
|
108
|
+
return;
|
|
109
|
+
onError(error);
|
|
110
|
+
relisten();
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
function drain() {
|
|
114
|
+
if (stopped)
|
|
115
|
+
return;
|
|
116
|
+
if (draining) {
|
|
117
|
+
again = true;
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
draining = (async () => {
|
|
121
|
+
do {
|
|
122
|
+
again = false;
|
|
123
|
+
try {
|
|
124
|
+
const rows = await db
|
|
125
|
+
.select(SIGNAL)
|
|
126
|
+
.from(controlSignals)
|
|
127
|
+
.where(gte(controlSignals.createdAt, sql `greatest(${since}::timestamptz, clock_timestamp() - make_interval(secs => ${WINDOW_SECONDS}))`))
|
|
128
|
+
.orderBy(controlSignals.id);
|
|
129
|
+
for (const row of rows)
|
|
130
|
+
deliver(row);
|
|
131
|
+
}
|
|
132
|
+
catch (error) {
|
|
133
|
+
if (!stopped)
|
|
134
|
+
onError(error);
|
|
135
|
+
}
|
|
136
|
+
forgetOld();
|
|
137
|
+
} while (again && !stopped);
|
|
138
|
+
})().finally(() => {
|
|
139
|
+
draining = undefined;
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
function deliver(row) {
|
|
143
|
+
if (stopped || delivered.has(row.id))
|
|
144
|
+
return;
|
|
145
|
+
delivered.set(row.id, Date.now());
|
|
146
|
+
const signal = signalOf(row);
|
|
147
|
+
if (!signal)
|
|
148
|
+
return;
|
|
149
|
+
try {
|
|
150
|
+
onSignal(signal);
|
|
151
|
+
}
|
|
152
|
+
catch (error) {
|
|
153
|
+
onError(error);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
/** A row a read-back can still return is younger than the window: keep twice that. */
|
|
157
|
+
function forgetOld() {
|
|
158
|
+
const before = Date.now() - 2 * WINDOW_SECONDS * 1000;
|
|
159
|
+
for (const [id, at] of delivered)
|
|
160
|
+
if (at < before)
|
|
161
|
+
delivered.delete(id);
|
|
162
|
+
}
|
|
163
|
+
const poll = setInterval(drain, pollMs);
|
|
164
|
+
poll.unref();
|
|
165
|
+
await listen();
|
|
166
|
+
return {
|
|
167
|
+
async close() {
|
|
168
|
+
stopped = true;
|
|
169
|
+
clearInterval(poll);
|
|
170
|
+
if (retry)
|
|
171
|
+
clearTimeout(retry);
|
|
172
|
+
await draining;
|
|
173
|
+
await client.end({ timeout: 0 });
|
|
174
|
+
},
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
function signalOf(row) {
|
|
178
|
+
if (row.kind === "session.cancel" && row.sessionId !== null)
|
|
179
|
+
return {
|
|
180
|
+
type: "session.cancel",
|
|
181
|
+
sessionId: row.sessionId,
|
|
182
|
+
...(row.turnId !== null ? { turnId: row.turnId } : {}),
|
|
183
|
+
};
|
|
184
|
+
if (row.kind === "sessions.reset" && row.generation !== null)
|
|
185
|
+
return { type: "sessions.reset", generation: row.generation };
|
|
186
|
+
return undefined;
|
|
187
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
CREATE TABLE "nylorun"."control_signals" (
|
|
2
|
+
"id" bigint PRIMARY KEY GENERATED ALWAYS AS IDENTITY (sequence name "nylorun"."control_signals_id_seq" INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 CACHE 1),
|
|
3
|
+
"kind" text NOT NULL,
|
|
4
|
+
"session_id" text COLLATE "C",
|
|
5
|
+
"turn_id" text COLLATE "C",
|
|
6
|
+
"generation" integer,
|
|
7
|
+
"created_at" timestamp with time zone DEFAULT clock_timestamp() NOT NULL,
|
|
8
|
+
CONSTRAINT "control_signals_kind_check" CHECK ("nylorun"."control_signals"."kind" IN ('session.cancel', 'sessions.reset'))
|
|
9
|
+
);
|
|
10
|
+
--> statement-breakpoint
|
|
11
|
+
CREATE INDEX "control_signals_created_at" ON "nylorun"."control_signals" USING btree ("created_at");
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
DROP TABLE "nylorun"."oauth_pending" CASCADE;--> statement-breakpoint
|
|
2
|
+
INSERT INTO "nylorun"."vault_audit" ("id", "at", "actor", "action", "vault_id", "credential_id", "target", "outcome")
|
|
3
|
+
SELECT gen_random_uuid()::text, to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'), 'migration', 'delete', "vault_id", "id", "binding_json"::jsonb ->> 'url', 'deleted'
|
|
4
|
+
FROM "nylorun"."vault_credentials" WHERE "type" = 'oauth' ORDER BY "vault_id", "id";--> statement-breakpoint
|
|
5
|
+
DELETE FROM "nylorun"."vault_credentials" WHERE "type" = 'oauth';
|
|
@@ -106,6 +106,20 @@
|
|
|
106
106
|
"when": 1791218903443,
|
|
107
107
|
"tag": "0014_action_endpoints_removed",
|
|
108
108
|
"breakpoints": true
|
|
109
|
+
},
|
|
110
|
+
{
|
|
111
|
+
"idx": 15,
|
|
112
|
+
"version": "7",
|
|
113
|
+
"when": 1791296131735,
|
|
114
|
+
"tag": "0015_control_signals",
|
|
115
|
+
"breakpoints": true
|
|
116
|
+
},
|
|
117
|
+
{
|
|
118
|
+
"idx": 16,
|
|
119
|
+
"version": "7",
|
|
120
|
+
"when": 1791395759051,
|
|
121
|
+
"tag": "0016_mcp_oauth_removed",
|
|
122
|
+
"breakpoints": true
|
|
109
123
|
}
|
|
110
124
|
]
|
|
111
125
|
}
|