@turri-dev/cli 0.2.0 → 0.2.1-next.47b578c
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/dist/chats.d.ts +31 -4
- package/dist/chats.js +146 -52
- package/dist/cli.js +4 -2
- package/dist/config.d.ts +4 -1
- package/dist/config.js +39 -4
- package/dist/environment.d.ts +0 -2
- package/dist/environment.js +0 -6
- package/dist/extension.d.ts +4 -3
- package/dist/extension.js +110 -23
- package/dist/host-session.d.ts +14 -0
- package/dist/host-session.js +2 -0
- package/dist/host.d.ts +6 -1
- package/dist/host.js +5 -1
- package/dist/init-config.d.ts +1 -1
- package/dist/init-config.js +33 -3
- package/dist/installed-providers.d.ts +40 -0
- package/dist/installed-providers.js +98 -0
- package/dist/model-check.d.ts +4 -2
- package/dist/model-check.js +33 -17
- package/dist/notice.d.ts +30 -6
- package/dist/pi.js +8 -24
- package/dist/plain/files.d.ts +16 -0
- package/dist/plain/files.js +68 -5
- package/dist/plain/graph.js +2 -2
- package/dist/plain/module-concepts.js +3 -2
- package/dist/plain/parser.d.ts +8 -3
- package/dist/plain/parser.js +47 -6
- package/dist/plain/stdlib.js +3 -3
- package/dist/regenerate/base-folder.d.ts +2 -0
- package/dist/regenerate/base-folder.js +24 -6
- package/dist/regenerate/command-check.d.ts +12 -1
- package/dist/regenerate/command-check.js +74 -0
- package/dist/regenerate/confine.d.ts +5 -0
- package/dist/regenerate/confine.js +9 -9
- package/dist/regenerate/ledger.d.ts +7 -0
- package/dist/regenerate/lineage.js +43 -2
- package/dist/regenerate/loop.d.ts +8 -0
- package/dist/regenerate/loop.js +11 -4
- package/dist/regenerate/os-sandbox.d.ts +2 -0
- package/dist/regenerate/os-sandbox.js +55 -20
- package/dist/regenerate/release.d.ts +2 -0
- package/dist/regenerate/requires.d.ts +12 -0
- package/dist/regenerate/requires.js +70 -1
- package/dist/regenerate/spec-steps.js +18 -1
- package/dist/regenerate/standing.d.ts +28 -0
- package/dist/regenerate/standing.js +30 -0
- package/dist/regenerate/testing.d.ts +2 -0
- package/dist/regenerate/testing.js +14 -0
- package/dist/release-agent.d.ts +17 -0
- package/dist/release-agent.js +19 -7
- package/dist/release-tools.d.ts +6 -0
- package/dist/release-tools.js +24 -5
- package/dist/sentinel-chat.d.ts +1 -1
- package/dist/sentinel-chat.js +21 -8
- package/dist/sentinel-check.d.ts +36 -0
- package/dist/sentinel-check.js +78 -0
- package/dist/sentinel.d.ts +40 -9
- package/dist/sentinel.js +240 -37
- package/dist/setup.d.ts +24 -3
- package/dist/setup.js +117 -31
- package/dist/spawn-async.d.ts +18 -0
- package/dist/spawn-async.js +35 -0
- package/dist/stack.d.ts +14 -0
- package/dist/stack.js +44 -2
- package/dist/turri-versions.d.ts +71 -0
- package/dist/turri-versions.js +275 -0
- package/docs/README.md +2 -0
- package/docs/claude-subscription.md +131 -0
- package/docs/config.md +5 -0
- package/docs/deploy.md +2 -2
- package/docs/hosting.md +42 -4
- package/docs/installation.md +39 -2
- package/docs/parser.md +1 -1
- package/docs/regenerating.md +1 -1
- package/docs/sandbox.md +2 -1
- package/docs/sentinel.md +11 -3
- package/docs/using.md +4 -2
- package/genome/rules/turri-regeneration.md +1 -1
- package/package.json +1 -1
package/dist/model-check.d.ts
CHANGED
|
@@ -10,6 +10,8 @@ import { type ProjectConfig } from "./config.ts";
|
|
|
10
10
|
export interface UsedModel {
|
|
11
11
|
spec: string;
|
|
12
12
|
usedBy: string[];
|
|
13
|
+
/** How hard it thinks, as the first setting naming it says - none written, as the agents run it: low at least. */
|
|
14
|
+
thinking?: string;
|
|
13
15
|
}
|
|
14
16
|
/** Every model the config uses, each once, with the settings that name it - the two models first. */
|
|
15
17
|
export declare function modelsInUse(config: Pick<ProjectConfig, "harnessModel" | "codingModel" | "judgeModel" | "models">): UsedModel[];
|
|
@@ -31,8 +33,8 @@ export interface ModelAccess {
|
|
|
31
33
|
find(provider: string, id: string): unknown;
|
|
32
34
|
/** Other models of the provider pi knows, to suggest from. */
|
|
33
35
|
known(provider: string): string[];
|
|
34
|
-
/** One short request: the error the provider gave, or nothing when it answered. */
|
|
35
|
-
ask(model: unknown): Promise<string | undefined>;
|
|
36
|
+
/** One short request, thinking as hard as `thinking` says: the error the provider gave, or nothing when it answered. */
|
|
37
|
+
ask(model: unknown, thinking?: string): Promise<string | undefined>;
|
|
36
38
|
}
|
|
37
39
|
/** A model in pi's catalogue: what turri chooses among, and checks a model it is given against. */
|
|
38
40
|
export interface CatalogueModel {
|
package/dist/model-check.js
CHANGED
|
@@ -1,31 +1,37 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { ModelRegistry } from "@earendil-works/pi-coding-agent";
|
|
2
|
+
import { EXTENSION_PROVIDERS, installedProviders, runtimeWithProviders } from "./installed-providers.js";
|
|
3
3
|
import { AGENT_ROLES, projectModels, roleModel } from "./config.js";
|
|
4
4
|
/** Every model the config uses, each once, with the settings that name it - the two models first. */
|
|
5
5
|
export function modelsInUse(config) {
|
|
6
6
|
const used = new Map();
|
|
7
|
-
const
|
|
7
|
+
const thinking = new Map();
|
|
8
|
+
const add = (choice, by) => {
|
|
9
|
+
const spec = choice?.model;
|
|
8
10
|
if (spec === undefined)
|
|
9
11
|
return;
|
|
10
12
|
used.set(spec, [...(used.get(spec) ?? []), by]);
|
|
13
|
+
if (!thinking.has(spec) && choice?.thinking !== undefined)
|
|
14
|
+
thinking.set(spec, choice.thinking);
|
|
11
15
|
};
|
|
12
16
|
const { harness, coding } = projectModels(config);
|
|
13
|
-
add(harness
|
|
14
|
-
add(coding
|
|
15
|
-
add(config.judgeModel
|
|
17
|
+
add(harness, "harness-model");
|
|
18
|
+
add(coding, "coding-model");
|
|
19
|
+
add(config.judgeModel, "judge-model");
|
|
16
20
|
// A role named under models: runs on its own model - or what it inherits, when its entry names only how it thinks.
|
|
17
21
|
for (const role of AGENT_ROLES) {
|
|
18
22
|
if (config.models?.[role] === undefined || role === "chat" || role === "coding")
|
|
19
23
|
continue;
|
|
20
|
-
add(roleModel(config, role)
|
|
24
|
+
add(roleModel(config, role), `models.${role}`);
|
|
21
25
|
}
|
|
22
|
-
add(config.models?.default
|
|
23
|
-
return [...used].map(([spec, usedBy]) => ({ spec, usedBy: [...new Set(usedBy)] }));
|
|
26
|
+
add(config.models?.default, "models.default");
|
|
27
|
+
return [...used].map(([spec, usedBy]) => ({ spec, usedBy: [...new Set(usedBy)], ...(thinking.has(spec) ? { thinking: thinking.get(spec) } : {}) }));
|
|
24
28
|
}
|
|
25
|
-
/**
|
|
29
|
+
/**
|
|
30
|
+
* pi's own model runtime: its catalogue with models.json's, pi's credentials - auth.json and the environment - and the
|
|
31
|
+
* providers the person's installed pi extensions register (installed-providers.ts), as turri's agents have them.
|
|
32
|
+
*/
|
|
26
33
|
async function piRuntime() {
|
|
27
|
-
|
|
28
|
-
return ModelRuntime.create({ authPath: join(agentDir, "auth.json"), modelsPath: join(agentDir, "models.json") });
|
|
34
|
+
return runtimeWithProviders(await installedProviders());
|
|
29
35
|
}
|
|
30
36
|
/** pi's catalogue, read once: no request, no model asked. */
|
|
31
37
|
export async function piModelCatalogue() {
|
|
@@ -67,9 +73,12 @@ export async function piModelAccess() {
|
|
|
67
73
|
return {
|
|
68
74
|
find: (provider, id) => registry.find(provider, id),
|
|
69
75
|
known: (provider) => runtime.getModels(provider).map((model) => model.id),
|
|
70
|
-
async ask(model) {
|
|
76
|
+
async ask(model, thinking) {
|
|
77
|
+
// As the agents ask it: thinking as the config says - some models refuse a request with none, Claude Code's among
|
|
78
|
+
// them - with room for the thinking before the word.
|
|
79
|
+
const reasoning = thinking !== undefined && thinking !== "off" && model.reasoning === true ? thinking : undefined;
|
|
71
80
|
try {
|
|
72
|
-
const reply = await runtime.completeSimple(model, { messages: [{ role: "user", content: "Reply with the one word: ok", timestamp: Date.now() }] }, { maxTokens: 64, signal: AbortSignal.timeout(60_000) });
|
|
81
|
+
const reply = await runtime.completeSimple(model, { messages: [{ role: "user", content: "Reply with the one word: ok", timestamp: Date.now() }] }, { maxTokens: reasoning !== undefined ? 2048 : 64, ...(reasoning !== undefined ? { reasoning: reasoning } : {}), signal: AbortSignal.timeout(60_000) });
|
|
73
82
|
return reply.stopReason === "error" || reply.stopReason === "aborted" ? (reply.errorMessage ?? `the request ended: ${reply.stopReason}`) : undefined;
|
|
74
83
|
}
|
|
75
84
|
catch (error) {
|
|
@@ -85,6 +94,7 @@ function signIn(provider) {
|
|
|
85
94
|
openai: "OPENAI_API_KEY, or /login in turri",
|
|
86
95
|
google: "GEMINI_API_KEY",
|
|
87
96
|
"amazon-bedrock": "AWS_PROFILE (aws configure, or aws sso login) or AWS_BEARER_TOKEN_BEDROCK",
|
|
97
|
+
...Object.fromEntries(Object.entries(EXTENSION_PROVIDERS).map(([id, provider]) => [id, provider.signIn])),
|
|
88
98
|
};
|
|
89
99
|
return keys[provider] ?? `the key pi reads for ${provider}, or /login in turri - https://pi.dev lists them`;
|
|
90
100
|
}
|
|
@@ -123,7 +133,7 @@ function commonPrefix(a, b) {
|
|
|
123
133
|
}
|
|
124
134
|
/** Each model the config uses, tried: answered, or what is wrong and what to do. One request a model. */
|
|
125
135
|
export async function checkModels(models, access, options = {}) {
|
|
126
|
-
return Promise.all(models.map(async ({ spec, usedBy }) => {
|
|
136
|
+
return Promise.all(models.map(async ({ spec, usedBy, thinking }) => {
|
|
127
137
|
const slash = spec.indexOf("/");
|
|
128
138
|
if (slash === -1)
|
|
129
139
|
return { ok: false, spec, usedBy, problem: "not provider/model-id", fix: "write it as provider/model-id - amazon-bedrock/global.openai.gpt-6-astra" };
|
|
@@ -131,13 +141,19 @@ export async function checkModels(models, access, options = {}) {
|
|
|
131
141
|
const model = access.find(provider, id);
|
|
132
142
|
if (model === undefined) {
|
|
133
143
|
const known = access.known(provider);
|
|
134
|
-
if (known.length === 0)
|
|
144
|
+
if (known.length === 0) {
|
|
145
|
+
// A provider an extension brings: its extension is not installed - or, installed, not ready.
|
|
146
|
+
const extension = EXTENSION_PROVIDERS[provider];
|
|
147
|
+
if (extension !== undefined)
|
|
148
|
+
return { ok: false, spec, usedBy, problem: `${provider} - ${extension.label} - is not there`, fix: `install its pi extension: ${extension.install} - with ${extension.signIn}` };
|
|
135
149
|
return { ok: false, spec, usedBy, problem: `pi knows no provider ${provider}`, fix: "use a provider pi supports - https://pi.dev" };
|
|
150
|
+
}
|
|
136
151
|
const close = closestModels(id, known);
|
|
137
152
|
return { ok: false, spec, usedBy, problem: `pi knows no model ${id} at ${provider}`, fix: close.length > 0 ? `did you mean ${close.map((other) => `${provider}/${other}`).join(", ")}?` : `change it in config.turri.yaml to a model pi knows at ${provider}` };
|
|
138
153
|
}
|
|
139
154
|
const started = Date.now();
|
|
140
|
-
|
|
155
|
+
// How the agents run it: as the config says, else low - a role's own default is at least that.
|
|
156
|
+
const problem = await access.ask(model, thinking ?? "low");
|
|
141
157
|
if (problem === undefined)
|
|
142
158
|
return { ok: true, spec, usedBy, ms: Date.now() - started };
|
|
143
159
|
const said = problem.split("\n").find((line) => line.trim() !== "")?.trim().slice(0, 300) ?? problem;
|
package/dist/notice.d.ts
CHANGED
|
@@ -15,9 +15,11 @@ export interface NoticeAction {
|
|
|
15
15
|
export interface NoticeInput {
|
|
16
16
|
/**
|
|
17
17
|
* `sentinel-start`: not something wrong but a question turri puts before the sentinel first opens in a session - its
|
|
18
|
-
* actions start it, or not, and are turri's to carry out, not the chat's agent's (SENTINEL_START).
|
|
18
|
+
* actions start it, or not, and are turri's to carry out, not the chat's agent's (SENTINEL_START). `fixed`: the
|
|
19
|
+
* sentinel says what it fixed itself in the project's tooling - a testing script, its config - with nothing to answer:
|
|
20
|
+
* a note, never put as a question, and never superseding a notice about its module. None: the sentinel's.
|
|
19
21
|
*/
|
|
20
|
-
kind?: "sentinel-start";
|
|
22
|
+
kind?: "sentinel-start" | "fixed";
|
|
21
23
|
severity: NoticeSeverity;
|
|
22
24
|
/** What is wrong, in a line. */
|
|
23
25
|
summary: string;
|
|
@@ -29,6 +31,8 @@ export interface NoticeInput {
|
|
|
29
31
|
job?: string;
|
|
30
32
|
/** Its number among its module's pulses. */
|
|
31
33
|
pulse?: number;
|
|
34
|
+
/** The check it is about, when a host asked the sentinel to check functional specs (`SentinelCheck`): its id. */
|
|
35
|
+
check?: string;
|
|
32
36
|
/**
|
|
33
37
|
* The person may leave it as the build has it, for good (`{ ignore: true }`): a gap in the ***plain the build settled
|
|
34
38
|
* with an assumption - never something broken, a pulse that went wrong, a release that did not come up, nor the
|
|
@@ -79,12 +83,32 @@ export declare function ignoredGuidance(notice: Notice): string;
|
|
|
79
83
|
export declare function describeNotice(notice: NoticeInput): string;
|
|
80
84
|
/** Whether a pulse went wrong: failed, stopped, or ran to its end with what it built not passing. */
|
|
81
85
|
export declare function wentWrong(job: Job): boolean;
|
|
82
|
-
/**
|
|
86
|
+
/**
|
|
87
|
+
* The sentinel done with a pulse - or a check a host asked for: whether what it checked holds, and what it checked and
|
|
88
|
+
* found, in a line or two. `unchecked`: a check that never ran - the person did not start the sentinel, or it could not
|
|
89
|
+
* be opened - turri's own word, never the sentinel's.
|
|
90
|
+
*/
|
|
83
91
|
export interface Conclusion {
|
|
84
|
-
/** The pulse: its job id. */
|
|
85
|
-
job
|
|
86
|
-
|
|
92
|
+
/** The pulse: its job id. Undefined for a check. */
|
|
93
|
+
job?: string;
|
|
94
|
+
/** The check: its id (`SentinelCheck`). Undefined for a pulse. */
|
|
95
|
+
check?: string;
|
|
96
|
+
verdict: "holds" | "wrong" | "unchecked";
|
|
87
97
|
summary: string;
|
|
88
98
|
module?: string;
|
|
89
99
|
pulse?: number;
|
|
90
100
|
}
|
|
101
|
+
/**
|
|
102
|
+
* Functional specs of a module a host asked the sentinel to check, in the module's release: its last commit, released,
|
|
103
|
+
* and every module its requires chain builds on with it. Queued with the pulses, in a conversation of its own.
|
|
104
|
+
*/
|
|
105
|
+
export interface SentinelCheck {
|
|
106
|
+
id: string;
|
|
107
|
+
/** The module released and tested, by name. */
|
|
108
|
+
module: string;
|
|
109
|
+
plainFile: string;
|
|
110
|
+
/** The functional specs to test, each `module:func-id` - of the module, or of a module its requires chain builds on. */
|
|
111
|
+
funcIds: string[];
|
|
112
|
+
/** When it was asked for. */
|
|
113
|
+
at: string;
|
|
114
|
+
}
|
package/dist/pi.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { basename, dirname, join, resolve, sep } from "node:path";
|
|
2
2
|
import { fileURLToPath } from "node:url";
|
|
3
|
-
import { createAgentSession, DefaultResourceLoader, getAgentDir, ModelRegistry,
|
|
3
|
+
import { createAgentSession, DefaultResourceLoader, getAgentDir, ModelRegistry, SessionManager, } from "@earendil-works/pi-coding-agent";
|
|
4
|
+
import { installedProviders, runtimeWithProviders } from "./installed-providers.js";
|
|
4
5
|
import { loadRules, renderRulesBlock } from "./rules.js";
|
|
5
6
|
import { debugMode, inheritModel, stackOf, readProjectConfig, ROLE_THINKING, roleModel, roleTurnLimitMs, splitModelSpec } from "./config.js";
|
|
6
7
|
import { maskToolResults } from "./regenerate/masking.js";
|
|
@@ -53,36 +54,19 @@ let providerSource;
|
|
|
53
54
|
export function inheritProviders(source) {
|
|
54
55
|
providerSource = source;
|
|
55
56
|
}
|
|
56
|
-
/**
|
|
57
|
+
/**
|
|
58
|
+
* A model runtime for a sub-agent: pi's configuration, with the providers the conversation's extensions registered - or,
|
|
59
|
+
* with no conversation to take them from (a command run headless), those the person's installed pi extensions register.
|
|
60
|
+
*/
|
|
57
61
|
async function subAgentModelRuntime() {
|
|
58
|
-
const agentDir = getAgentDir();
|
|
59
|
-
const runtime = await ModelRuntime.create({ authPath: join(agentDir, "auth.json"), modelsPath: join(agentDir, "models.json") });
|
|
60
62
|
let source;
|
|
61
63
|
try {
|
|
62
64
|
source = providerSource?.();
|
|
63
65
|
}
|
|
64
66
|
catch {
|
|
65
|
-
// The conversation it came from is gone:
|
|
66
|
-
}
|
|
67
|
-
if (source === undefined)
|
|
68
|
-
return runtime;
|
|
69
|
-
const registry = new ModelRegistry(runtime);
|
|
70
|
-
for (const id of source.getRegisteredProviderIds()) {
|
|
71
|
-
const native = source.getRegisteredNativeProvider(id);
|
|
72
|
-
const config = source.getRegisteredProviderConfig(id);
|
|
73
|
-
try {
|
|
74
|
-
if (native !== undefined)
|
|
75
|
-
registry.registerProvider(native);
|
|
76
|
-
else if (config !== undefined)
|
|
77
|
-
registry.registerProvider(id, config);
|
|
78
|
-
}
|
|
79
|
-
catch {
|
|
80
|
-
// One provider that will not register leaves the others, and pi's own, usable.
|
|
81
|
-
}
|
|
67
|
+
// The conversation it came from is gone: the installed ones, then.
|
|
82
68
|
}
|
|
83
|
-
|
|
84
|
-
await runtime.refresh({ allowNetwork: false });
|
|
85
|
-
return runtime;
|
|
69
|
+
return runtimeWithProviders(source ?? (await installedProviders()));
|
|
86
70
|
}
|
|
87
71
|
const RULES_DIR = join(dirname(dirname(fileURLToPath(import.meta.url))), "genome", "rules");
|
|
88
72
|
/**
|
package/dist/plain/files.d.ts
CHANGED
|
@@ -2,6 +2,21 @@
|
|
|
2
2
|
export declare function openFrom(dirs: string[], fileName: string): string | null;
|
|
3
3
|
/** The file `openFrom` reads: the first of the folders that has it. */
|
|
4
4
|
export declare function findIn(dirs: string[], fileName: string): string | null;
|
|
5
|
+
/** Whether there is a file at `path` - looked for as `findIn` looks, and noted as it is. */
|
|
6
|
+
export declare function isFile(path: string): boolean;
|
|
7
|
+
declare const FOUND: unique symbol;
|
|
8
|
+
declare const MISSING: unique symbol;
|
|
9
|
+
/** What a parse read, while `noteReads` runs: each file it looked for - its text, there, or not there. */
|
|
10
|
+
export type Reads = Map<string, string | typeof FOUND | typeof MISSING>;
|
|
11
|
+
/** `run`, with every file it looked for and read through these helpers noted - an inner run's in the outer's too. */
|
|
12
|
+
export declare function noteReads<T>(run: () => T): {
|
|
13
|
+
value: T;
|
|
14
|
+
reads: Reads;
|
|
15
|
+
};
|
|
16
|
+
/** Reads made before - a kept result's - counted in the run noting now, as if made again. */
|
|
17
|
+
export declare function addReads(into: Reads | undefined, reads: Reads): void;
|
|
18
|
+
/** Whether each file noted is still as it was: the same text, still there, still not there. */
|
|
19
|
+
export declare function readsHold(reads: Reads): boolean;
|
|
5
20
|
/**
|
|
6
21
|
* Port of loaders.relative_resource_path: the path of `target` in `resourceDir`, relative to `baseDir`, with `/` as
|
|
7
22
|
* separator. The folder part is computed lexically and the target is appended as written (`..` kept), so joining
|
|
@@ -29,3 +44,4 @@ export declare function liquidErrorMessage(error: Error & {
|
|
|
29
44
|
input?: string;
|
|
30
45
|
};
|
|
31
46
|
}, templateName?: string): string;
|
|
47
|
+
export {};
|
package/dist/plain/files.js
CHANGED
|
@@ -6,21 +6,84 @@ import { createEnvironment } from "./liquid.js";
|
|
|
6
6
|
/** Ports of the Python reference implementation's file helpers this parser needs. */
|
|
7
7
|
export function openFrom(dirs, fileName) {
|
|
8
8
|
const path = findIn(dirs, fileName);
|
|
9
|
-
|
|
9
|
+
if (path === null)
|
|
10
|
+
return null;
|
|
11
|
+
const text = readFileSync(path, "utf8");
|
|
12
|
+
noted?.set(resolve(path), text);
|
|
13
|
+
return text;
|
|
10
14
|
}
|
|
11
15
|
/** The file `openFrom` reads: the first of the folders that has it. */
|
|
12
16
|
export function findIn(dirs, fileName) {
|
|
13
17
|
for (const dir of dirs) {
|
|
14
18
|
const fullPath = join(dir, fileName);
|
|
19
|
+
if (isFile(fullPath))
|
|
20
|
+
return fullPath;
|
|
21
|
+
}
|
|
22
|
+
return null;
|
|
23
|
+
}
|
|
24
|
+
/** Whether there is a file at `path` - looked for as `findIn` looks, and noted as it is. */
|
|
25
|
+
export function isFile(path) {
|
|
26
|
+
let found;
|
|
27
|
+
try {
|
|
28
|
+
found = statSync(path).isFile();
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
found = false;
|
|
32
|
+
}
|
|
33
|
+
const at = resolve(path);
|
|
34
|
+
if (noted !== undefined && !noted.has(at))
|
|
35
|
+
noted.set(at, found ? FOUND : MISSING);
|
|
36
|
+
return found;
|
|
37
|
+
}
|
|
38
|
+
const FOUND = Symbol("found");
|
|
39
|
+
const MISSING = Symbol("missing");
|
|
40
|
+
let noted;
|
|
41
|
+
/** `run`, with every file it looked for and read through these helpers noted - an inner run's in the outer's too. */
|
|
42
|
+
export function noteReads(run) {
|
|
43
|
+
const outer = noted;
|
|
44
|
+
const mine = new Map();
|
|
45
|
+
noted = mine;
|
|
46
|
+
try {
|
|
47
|
+
return { value: run(), reads: mine };
|
|
48
|
+
}
|
|
49
|
+
finally {
|
|
50
|
+
noted = outer;
|
|
51
|
+
if (outer !== undefined)
|
|
52
|
+
addReads(outer, mine);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
/** Reads made before - a kept result's - counted in the run noting now, as if made again. */
|
|
56
|
+
export function addReads(into = noted, reads) {
|
|
57
|
+
if (into === undefined)
|
|
58
|
+
return;
|
|
59
|
+
for (const [path, what] of reads)
|
|
60
|
+
if (!into.has(path) || typeof what === "string")
|
|
61
|
+
into.set(path, what);
|
|
62
|
+
}
|
|
63
|
+
/** Whether each file noted is still as it was: the same text, still there, still not there. */
|
|
64
|
+
export function readsHold(reads) {
|
|
65
|
+
for (const [path, was] of reads) {
|
|
66
|
+
if (typeof was === "string") {
|
|
67
|
+
try {
|
|
68
|
+
if (readFileSync(path, "utf8") !== was)
|
|
69
|
+
return false;
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
return false;
|
|
73
|
+
}
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
let there;
|
|
15
77
|
try {
|
|
16
|
-
|
|
17
|
-
return fullPath;
|
|
78
|
+
there = statSync(path).isFile();
|
|
18
79
|
}
|
|
19
80
|
catch {
|
|
20
|
-
|
|
81
|
+
there = false;
|
|
21
82
|
}
|
|
83
|
+
if (there !== (was === FOUND))
|
|
84
|
+
return false;
|
|
22
85
|
}
|
|
23
|
-
return
|
|
86
|
+
return true;
|
|
24
87
|
}
|
|
25
88
|
/**
|
|
26
89
|
* Port of loaders.relative_resource_path: the path of `target` in `resourceDir`, relative to `baseDir`, with `/` as
|
package/dist/plain/graph.js
CHANGED
|
@@ -6,7 +6,7 @@ import { resourceFile } from "../regenerate/resources.js";
|
|
|
6
6
|
import { DEFAULT_CONCEPT_DEFINITIONS, DEFAULT_CONCEPTS, definitionText, extractConceptsFromDefinition, findConcepts } from "./concepts.js";
|
|
7
7
|
import { lineNumber, renderToMarkdown, renderTokens } from "./markdown.js";
|
|
8
8
|
import { attempt, namesIn, readModule } from "./module-concepts.js";
|
|
9
|
-
import { IMPORT_DIRECTIVE, linksIn, modulePlainFile,
|
|
9
|
+
import { IMPORT_DIRECTIVE, linksIn, modulePlainFile, sharedPlainFileParse, PLAIN_SOURCE_FILE_EXTENSION, plainFileParser, readPlainSourceMetadata, REQUIRES_DIRECTIVE, functionalitiesOf, sectionOf, withParseCache, } from "./parser.js";
|
|
10
10
|
import { DEFINITIONS, FUNCTIONAL_REQUIREMENTS, NON_FUNCTIONAL_REQUIREMENTS, TEST_REQUIREMENTS } from "./spec.js";
|
|
11
11
|
import { isStdlibModule } from "./stdlib.js";
|
|
12
12
|
import { discoverTestPlainFiles, readTestPlainFileText } from "./project.js";
|
|
@@ -161,7 +161,7 @@ function parseError(node, dirs) {
|
|
|
161
161
|
const imported = node.importedBy.length > 0 || isStdlibModule(node.module) || basename(dirname(node.file)) === "template";
|
|
162
162
|
try {
|
|
163
163
|
if (imported)
|
|
164
|
-
|
|
164
|
+
sharedPlainFileParse(node.module, dirs);
|
|
165
165
|
else
|
|
166
166
|
plainFileParser(`${node.module}${PLAIN_SOURCE_FILE_EXTENSION}`, dirs, { allowNoFunctionalities: true });
|
|
167
167
|
return undefined;
|
|
@@ -3,7 +3,7 @@ import { basename } from "node:path";
|
|
|
3
3
|
import { resolveTemplateDirs } from "../regenerate/paths.js";
|
|
4
4
|
import { DEFAULT_CONCEPT_DEFINITIONS, definitionText, extractConceptsFromDefinition } from "./concepts.js";
|
|
5
5
|
import { renderToMarkdown } from "./markdown.js";
|
|
6
|
-
import { exportedDefinitionsOf, IMPORT_DIRECTIVE,
|
|
6
|
+
import { exportedDefinitionsOf, IMPORT_DIRECTIVE, sharedPlainFileParse, parsePlainSource, PLAIN_SOURCE_FILE_EXTENSION, readPlainSourceMetadata, REQUIRES_DIRECTIVE, restoreStrippedLines, sectionOf, } from "./parser.js";
|
|
7
7
|
import { DEFINITIONS } from "./spec.js";
|
|
8
8
|
/**
|
|
9
9
|
* A module read for what it can name, as the parser builds its definitions: its own, its imports' (`import`), and its
|
|
@@ -11,7 +11,8 @@ import { DEFINITIONS } from "./spec.js";
|
|
|
11
11
|
* gives its own definitions read without its imports, and what of those it names is there.
|
|
12
12
|
*/
|
|
13
13
|
export function readModule(moduleName, plainFile, templateDirs) {
|
|
14
|
-
|
|
14
|
+
// Read, never changed: the parse kept for the file, not a copy of it.
|
|
15
|
+
const parse = (name) => attempt(() => sharedPlainFileParse(name, templateDirs));
|
|
15
16
|
const definitions = [];
|
|
16
17
|
const add = (result, nodes, module) => {
|
|
17
18
|
for (const node of nodes)
|
package/dist/plain/parser.d.ts
CHANGED
|
@@ -100,6 +100,11 @@ export declare function parsePlainSource(plainSourceText: string, codeVariables:
|
|
|
100
100
|
export declare function modulePlainFile(moduleName: string, templateDirs: string[]): string | null;
|
|
101
101
|
export declare function readModulePlainSource(moduleName: string, templateDirs: string[]): string;
|
|
102
102
|
export declare function parsePlainFile(moduleName: string, codeVariables: Record<string, Record<string, string>>, templateDirs: string[], importedModules: string[], modulesTrace: string[]): PlainFileParseResult;
|
|
103
|
+
/**
|
|
104
|
+
* A module's file parsed on its own - as `parsePlainFile(moduleName, {}, templateDirs, [], [])` parses it - shared with
|
|
105
|
+
* every caller, not a copy: for what only reads it. What builds on a parse asks `parsePlainFile` for its own copy.
|
|
106
|
+
*/
|
|
107
|
+
export declare function sharedPlainFileParse(moduleName: string, templateDirs: string[]): PlainFileParseResult;
|
|
103
108
|
/**
|
|
104
109
|
* Parse every required module, deepest first, without validating it - each with code variables of its own - and
|
|
105
110
|
* return the definitions the directly required modules export. A module listed again right after it was walked is
|
|
@@ -151,8 +156,8 @@ export declare function parseModuleChain(plainSourceFileName: string, templateDi
|
|
|
151
156
|
*
|
|
152
157
|
* Walking a requires chain parses each module in it, and each parse gathers what it inherits
|
|
153
158
|
* by parsing the modules below - so looking across a project of long chains asks for the
|
|
154
|
-
* same files thousands of times. Inside `run` they are parsed once.
|
|
155
|
-
*
|
|
156
|
-
*
|
|
159
|
+
* same files thousands of times. Inside `run` they are parsed once. Between calls only
|
|
160
|
+
* parses that still hold are kept - a top-level parse by its fingerprint, a file parsed on
|
|
161
|
+
* its own by every file it read - so an edit made between two calls is always seen.
|
|
157
162
|
*/
|
|
158
163
|
export declare function withParseCache<T>(run: () => T): T;
|
package/dist/plain/parser.js
CHANGED
|
@@ -4,7 +4,7 @@ import { extname, isAbsolute, basename, join, resolve } from "node:path";
|
|
|
4
4
|
import { load as loadYaml } from "js-yaml";
|
|
5
5
|
import { extractConceptsFromDefinition, findConceptDefinitionsInPlainSource, sortDefinitions, validateConcepts, DEFAULT_CONCEPTS, } from "./concepts.js";
|
|
6
6
|
import { ImportedModuleWithFunctionalitiesError, MissingFunctionalitiesError, ModuleDoesNotExistError, PlainSyntaxError, UnsupportedBase64Content, } from "./errors.js";
|
|
7
|
-
import { findIn, findLargeBase64Blob, getLoadedTemplates, openFrom } from "./files.js";
|
|
7
|
+
import { addReads, findIn, findLargeBase64Blob, getLoadedTemplates, noteReads, openFrom, readsHold } from "./files.js";
|
|
8
8
|
import { renderPlainSource } from "./liquid.js";
|
|
9
9
|
import { isStdlibModule, readStdlibModule, stdlibModuleFile } from "./stdlib.js";
|
|
10
10
|
import { getRawText, lineNumber, parseMarkdown, removeQuotes, renderToMarkdown, renderTokens, traverse, } from "./markdown.js";
|
|
@@ -547,6 +547,45 @@ export function readModulePlainSource(moduleName, templateDirs) {
|
|
|
547
547
|
return plainSourceText;
|
|
548
548
|
}
|
|
549
549
|
export function parsePlainFile(moduleName, codeVariables, templateDirs, importedModules, modulesTrace) {
|
|
550
|
+
// A file parsed on its own - not an import inside another's parse, whose result hangs on what was imported before it -
|
|
551
|
+
// is parsed once while nothing it read has changed: every module below a module in its chain parses each of them
|
|
552
|
+
// again, and a project of long chains asked the same files thousands of times.
|
|
553
|
+
const own = importedModules.length === 0 && modulesTrace.length === 0 && Object.keys(codeVariables).length === 0;
|
|
554
|
+
if (!own)
|
|
555
|
+
return parsePlainFileNow(moduleName, codeVariables, templateDirs, importedModules, modulesTrace);
|
|
556
|
+
// A copy: the caller builds on what it is given.
|
|
557
|
+
const copy = structuredClone(keptParse(moduleName, templateDirs));
|
|
558
|
+
Object.assign(codeVariables, copy.codeVariables);
|
|
559
|
+
return copy.result;
|
|
560
|
+
}
|
|
561
|
+
/** Files parsed on their own, with what each read - by working folder, template folders and module. Oldest out first. */
|
|
562
|
+
const keptFiles = new Map();
|
|
563
|
+
const KEPT_FILES = 500;
|
|
564
|
+
/**
|
|
565
|
+
* A module's file parsed on its own - as `parsePlainFile(moduleName, {}, templateDirs, [], [])` parses it - shared with
|
|
566
|
+
* every caller, not a copy: for what only reads it. What builds on a parse asks `parsePlainFile` for its own copy.
|
|
567
|
+
*/
|
|
568
|
+
export function sharedPlainFileParse(moduleName, templateDirs) {
|
|
569
|
+
return keptParse(moduleName, templateDirs).result;
|
|
570
|
+
}
|
|
571
|
+
/** A file parsed on its own, as kept while what it read holds - not to be changed: copy what is built on. */
|
|
572
|
+
function keptParse(moduleName, templateDirs) {
|
|
573
|
+
const key = `${process.cwd()}\0${templateDirs.join("\0")}\0${moduleName}`;
|
|
574
|
+
const kept = keptFiles.get(key);
|
|
575
|
+
if (kept !== undefined && readsHold(kept.reads)) {
|
|
576
|
+
addReads(undefined, kept.reads);
|
|
577
|
+
return kept.parsed;
|
|
578
|
+
}
|
|
579
|
+
const codeVariables = {};
|
|
580
|
+
const { value: result, reads } = noteReads(() => parsePlainFileNow(moduleName, codeVariables, templateDirs, [], []));
|
|
581
|
+
const parsed = { result, codeVariables };
|
|
582
|
+
keptFiles.delete(key);
|
|
583
|
+
keptFiles.set(key, { reads, parsed });
|
|
584
|
+
while (keptFiles.size > KEPT_FILES)
|
|
585
|
+
keptFiles.delete(keptFiles.keys().next().value);
|
|
586
|
+
return parsed;
|
|
587
|
+
}
|
|
588
|
+
function parsePlainFileNow(moduleName, codeVariables, templateDirs, importedModules, modulesTrace) {
|
|
550
589
|
const plainSourceText = readModulePlainSource(moduleName, templateDirs);
|
|
551
590
|
return parsePlainSource(plainSourceText, codeVariables, templateDirs, importedModules, [...modulesTrace, moduleName], moduleName);
|
|
552
591
|
}
|
|
@@ -567,7 +606,9 @@ export function processRequiredModules(requiredModules, templateDirs, allRequire
|
|
|
567
606
|
exportedDefinitions.push(...(exports.get(moduleName) ?? []));
|
|
568
607
|
continue;
|
|
569
608
|
}
|
|
570
|
-
|
|
609
|
+
// Only what it requires and what it exports are read: the kept parse, not a copy - but for what it exports, which the
|
|
610
|
+
// module requiring it takes into its own definitions.
|
|
611
|
+
const { result } = keptParse(moduleName, templateDirs);
|
|
571
612
|
if (result.requiredModules.length === 0) {
|
|
572
613
|
if (allRequiredModules.length > 0) {
|
|
573
614
|
// Required modules must form one chain; independent modules cannot yet be merged.
|
|
@@ -577,7 +618,7 @@ export function processRequiredModules(requiredModules, templateDirs, allRequire
|
|
|
577
618
|
else {
|
|
578
619
|
processRequiredModules(result.requiredModules, templateDirs, allRequiredModules, [...modulesTrace, moduleName], exports);
|
|
579
620
|
}
|
|
580
|
-
const own = exportedDefinitionsOf(result);
|
|
621
|
+
const own = structuredClone(exportedDefinitionsOf(result));
|
|
581
622
|
exports.set(moduleName, own);
|
|
582
623
|
exportedDefinitions.push(...own);
|
|
583
624
|
allRequiredModules.push(moduleName);
|
|
@@ -687,9 +728,9 @@ export function parseModuleChain(plainSourceFileName, templateDirs, options = {}
|
|
|
687
728
|
*
|
|
688
729
|
* Walking a requires chain parses each module in it, and each parse gathers what it inherits
|
|
689
730
|
* by parsing the modules below - so looking across a project of long chains asks for the
|
|
690
|
-
* same files thousands of times. Inside `run` they are parsed once.
|
|
691
|
-
*
|
|
692
|
-
*
|
|
731
|
+
* same files thousands of times. Inside `run` they are parsed once. Between calls only
|
|
732
|
+
* parses that still hold are kept - a top-level parse by its fingerprint, a file parsed on
|
|
733
|
+
* its own by every file it read - so an edit made between two calls is always seen.
|
|
693
734
|
*/
|
|
694
735
|
export function withParseCache(run) {
|
|
695
736
|
if (activeCache !== undefined)
|
package/dist/plain/stdlib.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { realpathSync } from "node:fs";
|
|
2
2
|
import { dirname, join } from "node:path";
|
|
3
3
|
import { fileURLToPath } from "node:url";
|
|
4
4
|
import { ModuleDoesNotExistError, PlainSyntaxError } from "./errors.js";
|
|
5
|
-
import { findIn, openFrom } from "./files.js";
|
|
5
|
+
import { findIn, isFile, openFrom } from "./files.js";
|
|
6
6
|
/**
|
|
7
7
|
* The ***plain standard library's names. `stdlib/` is a reserved prefix: `stdlib/backend/python-fastapi`
|
|
8
8
|
* is always turri's own template, in its `stdlib/` folder - never a module of the project's. A
|
|
@@ -36,7 +36,7 @@ export function readStdlibModule(moduleName, templateDirs) {
|
|
|
36
36
|
const own = join(STDLIB_ROOT, file);
|
|
37
37
|
for (const dir of templateDirs) {
|
|
38
38
|
const local = join(dir, file);
|
|
39
|
-
if (
|
|
39
|
+
if (isFile(local) && !sameFile(local, own)) {
|
|
40
40
|
throw new PlainSyntaxError(`Plain syntax error: '${name}' is a reserved name - modules under ${STDLIB_PREFIX} are turri's standard library - ` +
|
|
41
41
|
`but ${local} is a module of that name. Rename it, or the folder it is in.`);
|
|
42
42
|
}
|
|
@@ -31,6 +31,8 @@ export declare function baseFolderOf(plainFile: string): {
|
|
|
31
31
|
* does not ignore - so no dependencies or build output; otherwise every file but those of the folders never copied.
|
|
32
32
|
*/
|
|
33
33
|
export declare function baseFiles(folder: string): string[];
|
|
34
|
+
/** `baseFiles`, its git runs not holding the thread up - for doctor, on a host's thread. */
|
|
35
|
+
export declare function baseFilesAsync(folder: string): Promise<string[]>;
|
|
34
36
|
/** What a base folder holds now, as one hash: each file's path and bytes. */
|
|
35
37
|
export declare function hashBase(folder: string, files?: string[]): string;
|
|
36
38
|
/** What a codebase started from, when it started from a base folder. */
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { spawnSync } from "node:child_process";
|
|
2
|
+
import { spawnAsync } from "../spawn-async.js";
|
|
2
3
|
import { createHash } from "node:crypto";
|
|
3
4
|
import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
|
4
5
|
import { dirname, join, relative, resolve, sep } from "node:path";
|
|
@@ -34,14 +35,31 @@ const NOT_COPIED = new Set([".git", "node_modules", ".venv", "venv", "__pycache_
|
|
|
34
35
|
* does not ignore - so no dependencies or build output; otherwise every file but those of the folders never copied.
|
|
35
36
|
*/
|
|
36
37
|
export function baseFiles(folder) {
|
|
37
|
-
const inGit = spawnSync("git", ["-C", folder,
|
|
38
|
+
const inGit = spawnSync("git", ["-C", folder, ...IN_WORK_TREE], { encoding: "utf8" });
|
|
38
39
|
if (inGit.status === 0 && inGit.stdout.trim() === "true") {
|
|
39
|
-
const listed = spawnSync("git", ["-C", folder,
|
|
40
|
-
if (listed.status === 0)
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
40
|
+
const listed = spawnSync("git", ["-C", folder, ...LISTED], { encoding: "utf8", maxBuffer: 256 * 1024 * 1024 });
|
|
41
|
+
if (listed.status === 0)
|
|
42
|
+
return gitListed(folder, listed.stdout);
|
|
43
|
+
}
|
|
44
|
+
return walkedFiles(folder);
|
|
45
|
+
}
|
|
46
|
+
/** `baseFiles`, its git runs not holding the thread up - for doctor, on a host's thread. */
|
|
47
|
+
export async function baseFilesAsync(folder) {
|
|
48
|
+
const inGit = await spawnAsync("git", ["-C", folder, ...IN_WORK_TREE]);
|
|
49
|
+
if (inGit.status === 0 && inGit.stdout.trim() === "true") {
|
|
50
|
+
const listed = await spawnAsync("git", ["-C", folder, ...LISTED]);
|
|
51
|
+
if (listed.status === 0)
|
|
52
|
+
return gitListed(folder, listed.stdout);
|
|
44
53
|
}
|
|
54
|
+
return walkedFiles(folder);
|
|
55
|
+
}
|
|
56
|
+
const IN_WORK_TREE = ["rev-parse", "--is-inside-work-tree"];
|
|
57
|
+
const LISTED = ["ls-files", "-z", "--cached", "--others", "--exclude-standard", "--", "."];
|
|
58
|
+
/** What git listed, as files there: a tracked file deleted in the work tree is listed, and not there to copy. */
|
|
59
|
+
function gitListed(folder, listed) {
|
|
60
|
+
return [...new Set(listed.split("\0").filter((path) => path !== ""))].filter((path) => isFile(join(folder, path))).sort();
|
|
61
|
+
}
|
|
62
|
+
function walkedFiles(folder) {
|
|
45
63
|
const found = [];
|
|
46
64
|
const walk = (dir) => {
|
|
47
65
|
for (const entry of readdirSync(dir)) {
|
|
@@ -26,7 +26,7 @@ export interface CommandFence {
|
|
|
26
26
|
/** Whose paths these are: drive letters, backslashes and any case on Windows. */
|
|
27
27
|
platform?: "posix" | "win32";
|
|
28
28
|
}
|
|
29
|
-
export type CheckRule = "path" | "record" | "credential" | "install" | "kill";
|
|
29
|
+
export type CheckRule = "path" | "record" | "credential" | "install" | "system" | "kill";
|
|
30
30
|
export type CheckResult = {
|
|
31
31
|
ok: true;
|
|
32
32
|
} | {
|
|
@@ -36,5 +36,16 @@ export type CheckResult = {
|
|
|
36
36
|
};
|
|
37
37
|
/** What of the machine an agent's command may read, beside its roots and its toolchains: the system's own folders. */
|
|
38
38
|
export declare function machineReadable(platform?: NodeJS.Platform): string[];
|
|
39
|
+
/**
|
|
40
|
+
* Why a command line changes the machine - installs software outside the project, or changes a setting of the
|
|
41
|
+
* machine's or the person's environment - or nothing when it does not. The project's own dependencies, installed into
|
|
42
|
+
* the project (npm install, a venv, mvn, cargo), change nothing of the machine's. turri changes the machine nowhere: not
|
|
43
|
+
* an agent, not the chat, not a release - what the machine lacks is the person's to install, in their own words.
|
|
44
|
+
*/
|
|
45
|
+
export declare function machineChange(command: string): string | undefined;
|
|
46
|
+
/** The .plain file a command line writes - a ***plain module or a test plain file - or nothing when it writes none. */
|
|
47
|
+
export declare function writesPlain(command: string): string | undefined;
|
|
48
|
+
/** What turri says when a command would change the machine: what it does instead. */
|
|
49
|
+
export declare function machineChangeReason(why: string): string;
|
|
39
50
|
/** Check one command line against a role's fence: `{ ok: true }`, or which rule it breaks and what to do instead. */
|
|
40
51
|
export declare function checkCommand(command: string, fence: CommandFence): CheckResult;
|