@turri-dev/cli 0.1.28-next.d5f2680 → 0.2.1-next.47b578c

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +3 -2
  2. package/dist/chats.d.ts +31 -4
  3. package/dist/chats.js +146 -52
  4. package/dist/cli.js +4 -2
  5. package/dist/config.d.ts +4 -1
  6. package/dist/config.js +39 -4
  7. package/dist/environment.d.ts +0 -2
  8. package/dist/environment.js +0 -6
  9. package/dist/extension.d.ts +4 -3
  10. package/dist/extension.js +110 -23
  11. package/dist/host-session.d.ts +14 -0
  12. package/dist/host-session.js +2 -0
  13. package/dist/host.d.ts +6 -1
  14. package/dist/host.js +5 -1
  15. package/dist/init-config.d.ts +1 -1
  16. package/dist/init-config.js +33 -3
  17. package/dist/installed-providers.d.ts +40 -0
  18. package/dist/installed-providers.js +98 -0
  19. package/dist/model-check.d.ts +4 -2
  20. package/dist/model-check.js +33 -17
  21. package/dist/notice.d.ts +30 -6
  22. package/dist/pi.js +8 -24
  23. package/dist/plain/files.d.ts +16 -0
  24. package/dist/plain/files.js +68 -5
  25. package/dist/plain/graph.js +2 -2
  26. package/dist/plain/module-concepts.js +3 -2
  27. package/dist/plain/parser.d.ts +8 -3
  28. package/dist/plain/parser.js +47 -6
  29. package/dist/plain/stdlib.js +3 -3
  30. package/dist/regenerate/base-folder.d.ts +2 -0
  31. package/dist/regenerate/base-folder.js +24 -6
  32. package/dist/regenerate/command-check.d.ts +12 -1
  33. package/dist/regenerate/command-check.js +74 -0
  34. package/dist/regenerate/confine.d.ts +5 -0
  35. package/dist/regenerate/confine.js +9 -9
  36. package/dist/regenerate/ledger.d.ts +7 -0
  37. package/dist/regenerate/lineage.js +43 -2
  38. package/dist/regenerate/loop.d.ts +8 -0
  39. package/dist/regenerate/loop.js +11 -4
  40. package/dist/regenerate/os-sandbox.d.ts +2 -0
  41. package/dist/regenerate/os-sandbox.js +55 -20
  42. package/dist/regenerate/release.d.ts +2 -0
  43. package/dist/regenerate/requires.d.ts +12 -0
  44. package/dist/regenerate/requires.js +70 -1
  45. package/dist/regenerate/spec-steps.js +18 -1
  46. package/dist/regenerate/standing.d.ts +28 -0
  47. package/dist/regenerate/standing.js +30 -0
  48. package/dist/regenerate/testing.d.ts +2 -0
  49. package/dist/regenerate/testing.js +14 -0
  50. package/dist/release-agent.d.ts +17 -0
  51. package/dist/release-agent.js +19 -7
  52. package/dist/release-tools.d.ts +6 -0
  53. package/dist/release-tools.js +24 -5
  54. package/dist/sentinel-chat.d.ts +1 -1
  55. package/dist/sentinel-chat.js +21 -8
  56. package/dist/sentinel-check.d.ts +36 -0
  57. package/dist/sentinel-check.js +78 -0
  58. package/dist/sentinel.d.ts +40 -9
  59. package/dist/sentinel.js +240 -37
  60. package/dist/setup.d.ts +24 -3
  61. package/dist/setup.js +117 -31
  62. package/dist/spawn-async.d.ts +18 -0
  63. package/dist/spawn-async.js +35 -0
  64. package/dist/stack.d.ts +14 -0
  65. package/dist/stack.js +44 -2
  66. package/dist/turri-versions.d.ts +71 -0
  67. package/dist/turri-versions.js +275 -0
  68. package/docs/README.md +2 -0
  69. package/docs/claude-subscription.md +131 -0
  70. package/docs/config.md +5 -0
  71. package/docs/deploy.md +2 -2
  72. package/docs/hosting.md +42 -4
  73. package/docs/installation.md +39 -2
  74. package/docs/parser.md +1 -1
  75. package/docs/regenerating.md +1 -1
  76. package/docs/sandbox.md +2 -1
  77. package/docs/sentinel.md +11 -3
  78. package/docs/using.md +4 -2
  79. package/genome/rules/turri-regeneration.md +1 -1
  80. package/package.json +1 -1
@@ -10,6 +10,8 @@ import { type ProjectConfig } from "./config.ts";
10
10
  export interface UsedModel {
11
11
  spec: string;
12
12
  usedBy: string[];
13
+ /** How hard it thinks, as the first setting naming it says - none written, as the agents run it: low at least. */
14
+ thinking?: string;
13
15
  }
14
16
  /** Every model the config uses, each once, with the settings that name it - the two models first. */
15
17
  export declare function modelsInUse(config: Pick<ProjectConfig, "harnessModel" | "codingModel" | "judgeModel" | "models">): UsedModel[];
@@ -31,8 +33,8 @@ export interface ModelAccess {
31
33
  find(provider: string, id: string): unknown;
32
34
  /** Other models of the provider pi knows, to suggest from. */
33
35
  known(provider: string): string[];
34
- /** One short request: the error the provider gave, or nothing when it answered. */
35
- ask(model: unknown): Promise<string | undefined>;
36
+ /** One short request, thinking as hard as `thinking` says: the error the provider gave, or nothing when it answered. */
37
+ ask(model: unknown, thinking?: string): Promise<string | undefined>;
36
38
  }
37
39
  /** A model in pi's catalogue: what turri chooses among, and checks a model it is given against. */
38
40
  export interface CatalogueModel {
@@ -1,31 +1,37 @@
1
- import { join } from "node:path";
2
- import { getAgentDir, ModelRegistry, ModelRuntime } from "@earendil-works/pi-coding-agent";
1
+ import { ModelRegistry } from "@earendil-works/pi-coding-agent";
2
+ import { EXTENSION_PROVIDERS, installedProviders, runtimeWithProviders } from "./installed-providers.js";
3
3
  import { AGENT_ROLES, projectModels, roleModel } from "./config.js";
4
4
  /** Every model the config uses, each once, with the settings that name it - the two models first. */
5
5
  export function modelsInUse(config) {
6
6
  const used = new Map();
7
- const add = (spec, by) => {
7
+ const thinking = new Map();
8
+ const add = (choice, by) => {
9
+ const spec = choice?.model;
8
10
  if (spec === undefined)
9
11
  return;
10
12
  used.set(spec, [...(used.get(spec) ?? []), by]);
13
+ if (!thinking.has(spec) && choice?.thinking !== undefined)
14
+ thinking.set(spec, choice.thinking);
11
15
  };
12
16
  const { harness, coding } = projectModels(config);
13
- add(harness?.model, "harness-model");
14
- add(coding?.model, "coding-model");
15
- add(config.judgeModel?.model, "judge-model");
17
+ add(harness, "harness-model");
18
+ add(coding, "coding-model");
19
+ add(config.judgeModel, "judge-model");
16
20
  // A role named under models: runs on its own model - or what it inherits, when its entry names only how it thinks.
17
21
  for (const role of AGENT_ROLES) {
18
22
  if (config.models?.[role] === undefined || role === "chat" || role === "coding")
19
23
  continue;
20
- add(roleModel(config, role)?.model, `models.${role}`);
24
+ add(roleModel(config, role), `models.${role}`);
21
25
  }
22
- add(config.models?.default?.model, "models.default");
23
- return [...used].map(([spec, usedBy]) => ({ spec, usedBy: [...new Set(usedBy)] }));
26
+ add(config.models?.default, "models.default");
27
+ return [...used].map(([spec, usedBy]) => ({ spec, usedBy: [...new Set(usedBy)], ...(thinking.has(spec) ? { thinking: thinking.get(spec) } : {}) }));
24
28
  }
25
- /** pi's own model runtime: its catalogue with models.json's, and pi's credentials - auth.json and the environment. */
29
+ /**
30
+ * pi's own model runtime: its catalogue with models.json's, pi's credentials - auth.json and the environment - and the
31
+ * providers the person's installed pi extensions register (installed-providers.ts), as turri's agents have them.
32
+ */
26
33
  async function piRuntime() {
27
- const agentDir = getAgentDir();
28
- return ModelRuntime.create({ authPath: join(agentDir, "auth.json"), modelsPath: join(agentDir, "models.json") });
34
+ return runtimeWithProviders(await installedProviders());
29
35
  }
30
36
  /** pi's catalogue, read once: no request, no model asked. */
31
37
  export async function piModelCatalogue() {
@@ -67,9 +73,12 @@ export async function piModelAccess() {
67
73
  return {
68
74
  find: (provider, id) => registry.find(provider, id),
69
75
  known: (provider) => runtime.getModels(provider).map((model) => model.id),
70
- async ask(model) {
76
+ async ask(model, thinking) {
77
+ // As the agents ask it: thinking as the config says - some models refuse a request with none, Claude Code's among
78
+ // them - with room for the thinking before the word.
79
+ const reasoning = thinking !== undefined && thinking !== "off" && model.reasoning === true ? thinking : undefined;
71
80
  try {
72
- const reply = await runtime.completeSimple(model, { messages: [{ role: "user", content: "Reply with the one word: ok", timestamp: Date.now() }] }, { maxTokens: 64, signal: AbortSignal.timeout(60_000) });
81
+ const reply = await runtime.completeSimple(model, { messages: [{ role: "user", content: "Reply with the one word: ok", timestamp: Date.now() }] }, { maxTokens: reasoning !== undefined ? 2048 : 64, ...(reasoning !== undefined ? { reasoning: reasoning } : {}), signal: AbortSignal.timeout(60_000) });
73
82
  return reply.stopReason === "error" || reply.stopReason === "aborted" ? (reply.errorMessage ?? `the request ended: ${reply.stopReason}`) : undefined;
74
83
  }
75
84
  catch (error) {
@@ -85,6 +94,7 @@ function signIn(provider) {
85
94
  openai: "OPENAI_API_KEY, or /login in turri",
86
95
  google: "GEMINI_API_KEY",
87
96
  "amazon-bedrock": "AWS_PROFILE (aws configure, or aws sso login) or AWS_BEARER_TOKEN_BEDROCK",
97
+ ...Object.fromEntries(Object.entries(EXTENSION_PROVIDERS).map(([id, provider]) => [id, provider.signIn])),
88
98
  };
89
99
  return keys[provider] ?? `the key pi reads for ${provider}, or /login in turri - https://pi.dev lists them`;
90
100
  }
@@ -123,7 +133,7 @@ function commonPrefix(a, b) {
123
133
  }
124
134
  /** Each model the config uses, tried: answered, or what is wrong and what to do. One request a model. */
125
135
  export async function checkModels(models, access, options = {}) {
126
- return Promise.all(models.map(async ({ spec, usedBy }) => {
136
+ return Promise.all(models.map(async ({ spec, usedBy, thinking }) => {
127
137
  const slash = spec.indexOf("/");
128
138
  if (slash === -1)
129
139
  return { ok: false, spec, usedBy, problem: "not provider/model-id", fix: "write it as provider/model-id - amazon-bedrock/global.openai.gpt-6-astra" };
@@ -131,13 +141,19 @@ export async function checkModels(models, access, options = {}) {
131
141
  const model = access.find(provider, id);
132
142
  if (model === undefined) {
133
143
  const known = access.known(provider);
134
- if (known.length === 0)
144
+ if (known.length === 0) {
145
+ // A provider an extension brings: its extension is not installed - or, installed, not ready.
146
+ const extension = EXTENSION_PROVIDERS[provider];
147
+ if (extension !== undefined)
148
+ return { ok: false, spec, usedBy, problem: `${provider} - ${extension.label} - is not there`, fix: `install its pi extension: ${extension.install} - with ${extension.signIn}` };
135
149
  return { ok: false, spec, usedBy, problem: `pi knows no provider ${provider}`, fix: "use a provider pi supports - https://pi.dev" };
150
+ }
136
151
  const close = closestModels(id, known);
137
152
  return { ok: false, spec, usedBy, problem: `pi knows no model ${id} at ${provider}`, fix: close.length > 0 ? `did you mean ${close.map((other) => `${provider}/${other}`).join(", ")}?` : `change it in config.turri.yaml to a model pi knows at ${provider}` };
138
153
  }
139
154
  const started = Date.now();
140
- const problem = await access.ask(model);
155
+ // How the agents run it: as the config says, else low - a role's own default is at least that.
156
+ const problem = await access.ask(model, thinking ?? "low");
141
157
  if (problem === undefined)
142
158
  return { ok: true, spec, usedBy, ms: Date.now() - started };
143
159
  const said = problem.split("\n").find((line) => line.trim() !== "")?.trim().slice(0, 300) ?? problem;
package/dist/notice.d.ts CHANGED
@@ -15,9 +15,11 @@ export interface NoticeAction {
15
15
  export interface NoticeInput {
16
16
  /**
17
17
  * `sentinel-start`: not something wrong but a question turri puts before the sentinel first opens in a session - its
18
- * actions start it, or not, and are turri's to carry out, not the chat's agent's (SENTINEL_START). None: the sentinel's.
18
+ * actions start it, or not, and are turri's to carry out, not the chat's agent's (SENTINEL_START). `fixed`: the
19
+ * sentinel says what it fixed itself in the project's tooling - a testing script, its config - with nothing to answer:
20
+ * a note, never put as a question, and never superseding a notice about its module. None: the sentinel's.
19
21
  */
20
- kind?: "sentinel-start";
22
+ kind?: "sentinel-start" | "fixed";
21
23
  severity: NoticeSeverity;
22
24
  /** What is wrong, in a line. */
23
25
  summary: string;
@@ -29,6 +31,8 @@ export interface NoticeInput {
29
31
  job?: string;
30
32
  /** Its number among its module's pulses. */
31
33
  pulse?: number;
34
+ /** The check it is about, when a host asked the sentinel to check functional specs (`SentinelCheck`): its id. */
35
+ check?: string;
32
36
  /**
33
37
  * The person may leave it as the build has it, for good (`{ ignore: true }`): a gap in the ***plain the build settled
34
38
  * with an assumption - never something broken, a pulse that went wrong, a release that did not come up, nor the
@@ -79,12 +83,32 @@ export declare function ignoredGuidance(notice: Notice): string;
79
83
  export declare function describeNotice(notice: NoticeInput): string;
80
84
  /** Whether a pulse went wrong: failed, stopped, or ran to its end with what it built not passing. */
81
85
  export declare function wentWrong(job: Job): boolean;
82
- /** The sentinel done with a pulse: whether what it checked holds, and what it checked and found, in a line or two. */
86
+ /**
87
+ * The sentinel done with a pulse - or a check a host asked for: whether what it checked holds, and what it checked and
88
+ * found, in a line or two. `unchecked`: a check that never ran - the person did not start the sentinel, or it could not
89
+ * be opened - turri's own word, never the sentinel's.
90
+ */
83
91
  export interface Conclusion {
84
- /** The pulse: its job id. */
85
- job: string;
86
- verdict: "holds" | "wrong";
92
+ /** The pulse: its job id. Undefined for a check. */
93
+ job?: string;
94
+ /** The check: its id (`SentinelCheck`). Undefined for a pulse. */
95
+ check?: string;
96
+ verdict: "holds" | "wrong" | "unchecked";
87
97
  summary: string;
88
98
  module?: string;
89
99
  pulse?: number;
90
100
  }
101
+ /**
102
+ * Functional specs of a module a host asked the sentinel to check, in the module's release: its last commit, released,
103
+ * and every module its requires chain builds on with it. Queued with the pulses, in a conversation of its own.
104
+ */
105
+ export interface SentinelCheck {
106
+ id: string;
107
+ /** The module released and tested, by name. */
108
+ module: string;
109
+ plainFile: string;
110
+ /** The functional specs to test, each `module:func-id` - of the module, or of a module its requires chain builds on. */
111
+ funcIds: string[];
112
+ /** When it was asked for. */
113
+ at: string;
114
+ }
package/dist/pi.js CHANGED
@@ -1,6 +1,7 @@
1
1
  import { basename, dirname, join, resolve, sep } from "node:path";
2
2
  import { fileURLToPath } from "node:url";
3
- import { createAgentSession, DefaultResourceLoader, getAgentDir, ModelRegistry, ModelRuntime, SessionManager, } from "@earendil-works/pi-coding-agent";
3
+ import { createAgentSession, DefaultResourceLoader, getAgentDir, ModelRegistry, SessionManager, } from "@earendil-works/pi-coding-agent";
4
+ import { installedProviders, runtimeWithProviders } from "./installed-providers.js";
4
5
  import { loadRules, renderRulesBlock } from "./rules.js";
5
6
  import { debugMode, inheritModel, stackOf, readProjectConfig, ROLE_THINKING, roleModel, roleTurnLimitMs, splitModelSpec } from "./config.js";
6
7
  import { maskToolResults } from "./regenerate/masking.js";
@@ -53,36 +54,19 @@ let providerSource;
53
54
  export function inheritProviders(source) {
54
55
  providerSource = source;
55
56
  }
56
- /** A model runtime for a sub-agent: pi's configuration, with the providers the conversation's extensions registered. */
57
+ /**
58
+ * A model runtime for a sub-agent: pi's configuration, with the providers the conversation's extensions registered - or,
59
+ * with no conversation to take them from (a command run headless), those the person's installed pi extensions register.
60
+ */
57
61
  async function subAgentModelRuntime() {
58
- const agentDir = getAgentDir();
59
- const runtime = await ModelRuntime.create({ authPath: join(agentDir, "auth.json"), modelsPath: join(agentDir, "models.json") });
60
62
  let source;
61
63
  try {
62
64
  source = providerSource?.();
63
65
  }
64
66
  catch {
65
- // The conversation it came from is gone: pi's own configuration is what there is.
66
- }
67
- if (source === undefined)
68
- return runtime;
69
- const registry = new ModelRegistry(runtime);
70
- for (const id of source.getRegisteredProviderIds()) {
71
- const native = source.getRegisteredNativeProvider(id);
72
- const config = source.getRegisteredProviderConfig(id);
73
- try {
74
- if (native !== undefined)
75
- registry.registerProvider(native);
76
- else if (config !== undefined)
77
- registry.registerProvider(id, config);
78
- }
79
- catch {
80
- // One provider that will not register leaves the others, and pi's own, usable.
81
- }
67
+ // The conversation it came from is gone: the installed ones, then.
82
68
  }
83
- // Its models are listed as the registrations land; the agent's model is looked up right after.
84
- await runtime.refresh({ allowNetwork: false });
85
- return runtime;
69
+ return runtimeWithProviders(source ?? (await installedProviders()));
86
70
  }
87
71
  const RULES_DIR = join(dirname(dirname(fileURLToPath(import.meta.url))), "genome", "rules");
88
72
  /**
@@ -2,6 +2,21 @@
2
2
  export declare function openFrom(dirs: string[], fileName: string): string | null;
3
3
  /** The file `openFrom` reads: the first of the folders that has it. */
4
4
  export declare function findIn(dirs: string[], fileName: string): string | null;
5
+ /** Whether there is a file at `path` - looked for as `findIn` looks, and noted as it is. */
6
+ export declare function isFile(path: string): boolean;
7
+ declare const FOUND: unique symbol;
8
+ declare const MISSING: unique symbol;
9
+ /** What a parse read, while `noteReads` runs: each file it looked for - its text, there, or not there. */
10
+ export type Reads = Map<string, string | typeof FOUND | typeof MISSING>;
11
+ /** `run`, with every file it looked for and read through these helpers noted - an inner run's in the outer's too. */
12
+ export declare function noteReads<T>(run: () => T): {
13
+ value: T;
14
+ reads: Reads;
15
+ };
16
+ /** Reads made before - a kept result's - counted in the run noting now, as if made again. */
17
+ export declare function addReads(into: Reads | undefined, reads: Reads): void;
18
+ /** Whether each file noted is still as it was: the same text, still there, still not there. */
19
+ export declare function readsHold(reads: Reads): boolean;
5
20
  /**
6
21
  * Port of loaders.relative_resource_path: the path of `target` in `resourceDir`, relative to `baseDir`, with `/` as
7
22
  * separator. The folder part is computed lexically and the target is appended as written (`..` kept), so joining
@@ -29,3 +44,4 @@ export declare function liquidErrorMessage(error: Error & {
29
44
  input?: string;
30
45
  };
31
46
  }, templateName?: string): string;
47
+ export {};
@@ -6,21 +6,84 @@ import { createEnvironment } from "./liquid.js";
6
6
  /** Ports of the Python reference implementation's file helpers this parser needs. */
7
7
  export function openFrom(dirs, fileName) {
8
8
  const path = findIn(dirs, fileName);
9
- return path === null ? null : readFileSync(path, "utf8");
9
+ if (path === null)
10
+ return null;
11
+ const text = readFileSync(path, "utf8");
12
+ noted?.set(resolve(path), text);
13
+ return text;
10
14
  }
11
15
  /** The file `openFrom` reads: the first of the folders that has it. */
12
16
  export function findIn(dirs, fileName) {
13
17
  for (const dir of dirs) {
14
18
  const fullPath = join(dir, fileName);
19
+ if (isFile(fullPath))
20
+ return fullPath;
21
+ }
22
+ return null;
23
+ }
24
+ /** Whether there is a file at `path` - looked for as `findIn` looks, and noted as it is. */
25
+ export function isFile(path) {
26
+ let found;
27
+ try {
28
+ found = statSync(path).isFile();
29
+ }
30
+ catch {
31
+ found = false;
32
+ }
33
+ const at = resolve(path);
34
+ if (noted !== undefined && !noted.has(at))
35
+ noted.set(at, found ? FOUND : MISSING);
36
+ return found;
37
+ }
38
+ const FOUND = Symbol("found");
39
+ const MISSING = Symbol("missing");
40
+ let noted;
41
+ /** `run`, with every file it looked for and read through these helpers noted - an inner run's in the outer's too. */
42
+ export function noteReads(run) {
43
+ const outer = noted;
44
+ const mine = new Map();
45
+ noted = mine;
46
+ try {
47
+ return { value: run(), reads: mine };
48
+ }
49
+ finally {
50
+ noted = outer;
51
+ if (outer !== undefined)
52
+ addReads(outer, mine);
53
+ }
54
+ }
55
+ /** Reads made before - a kept result's - counted in the run noting now, as if made again. */
56
+ export function addReads(into = noted, reads) {
57
+ if (into === undefined)
58
+ return;
59
+ for (const [path, what] of reads)
60
+ if (!into.has(path) || typeof what === "string")
61
+ into.set(path, what);
62
+ }
63
+ /** Whether each file noted is still as it was: the same text, still there, still not there. */
64
+ export function readsHold(reads) {
65
+ for (const [path, was] of reads) {
66
+ if (typeof was === "string") {
67
+ try {
68
+ if (readFileSync(path, "utf8") !== was)
69
+ return false;
70
+ }
71
+ catch {
72
+ return false;
73
+ }
74
+ continue;
75
+ }
76
+ let there;
15
77
  try {
16
- if (statSync(fullPath).isFile())
17
- return fullPath;
78
+ there = statSync(path).isFile();
18
79
  }
19
80
  catch {
20
- continue;
81
+ there = false;
21
82
  }
83
+ if (there !== (was === FOUND))
84
+ return false;
22
85
  }
23
- return null;
86
+ return true;
24
87
  }
25
88
  /**
26
89
  * Port of loaders.relative_resource_path: the path of `target` in `resourceDir`, relative to `baseDir`, with `/` as
@@ -6,7 +6,7 @@ import { resourceFile } from "../regenerate/resources.js";
6
6
  import { DEFAULT_CONCEPT_DEFINITIONS, DEFAULT_CONCEPTS, definitionText, extractConceptsFromDefinition, findConcepts } from "./concepts.js";
7
7
  import { lineNumber, renderToMarkdown, renderTokens } from "./markdown.js";
8
8
  import { attempt, namesIn, readModule } from "./module-concepts.js";
9
- import { IMPORT_DIRECTIVE, linksIn, modulePlainFile, parsePlainFile, PLAIN_SOURCE_FILE_EXTENSION, plainFileParser, readPlainSourceMetadata, REQUIRES_DIRECTIVE, functionalitiesOf, sectionOf, withParseCache, } from "./parser.js";
9
+ import { IMPORT_DIRECTIVE, linksIn, modulePlainFile, sharedPlainFileParse, PLAIN_SOURCE_FILE_EXTENSION, plainFileParser, readPlainSourceMetadata, REQUIRES_DIRECTIVE, functionalitiesOf, sectionOf, withParseCache, } from "./parser.js";
10
10
  import { DEFINITIONS, FUNCTIONAL_REQUIREMENTS, NON_FUNCTIONAL_REQUIREMENTS, TEST_REQUIREMENTS } from "./spec.js";
11
11
  import { isStdlibModule } from "./stdlib.js";
12
12
  import { discoverTestPlainFiles, readTestPlainFileText } from "./project.js";
@@ -161,7 +161,7 @@ function parseError(node, dirs) {
161
161
  const imported = node.importedBy.length > 0 || isStdlibModule(node.module) || basename(dirname(node.file)) === "template";
162
162
  try {
163
163
  if (imported)
164
- parsePlainFile(node.module, {}, dirs, [], []);
164
+ sharedPlainFileParse(node.module, dirs);
165
165
  else
166
166
  plainFileParser(`${node.module}${PLAIN_SOURCE_FILE_EXTENSION}`, dirs, { allowNoFunctionalities: true });
167
167
  return undefined;
@@ -3,7 +3,7 @@ import { basename } from "node:path";
3
3
  import { resolveTemplateDirs } from "../regenerate/paths.js";
4
4
  import { DEFAULT_CONCEPT_DEFINITIONS, definitionText, extractConceptsFromDefinition } from "./concepts.js";
5
5
  import { renderToMarkdown } from "./markdown.js";
6
- import { exportedDefinitionsOf, IMPORT_DIRECTIVE, parsePlainFile, parsePlainSource, PLAIN_SOURCE_FILE_EXTENSION, readPlainSourceMetadata, REQUIRES_DIRECTIVE, restoreStrippedLines, sectionOf, } from "./parser.js";
6
+ import { exportedDefinitionsOf, IMPORT_DIRECTIVE, sharedPlainFileParse, parsePlainSource, PLAIN_SOURCE_FILE_EXTENSION, readPlainSourceMetadata, REQUIRES_DIRECTIVE, restoreStrippedLines, sectionOf, } from "./parser.js";
7
7
  import { DEFINITIONS } from "./spec.js";
8
8
  /**
9
9
  * A module read for what it can name, as the parser builds its definitions: its own, its imports' (`import`), and its
@@ -11,7 +11,8 @@ import { DEFINITIONS } from "./spec.js";
11
11
  * gives its own definitions read without its imports, and what of those it names is there.
12
12
  */
13
13
  export function readModule(moduleName, plainFile, templateDirs) {
14
- const parse = (name) => attempt(() => parsePlainFile(name, {}, templateDirs, [], []));
14
+ // Read, never changed: the parse kept for the file, not a copy of it.
15
+ const parse = (name) => attempt(() => sharedPlainFileParse(name, templateDirs));
15
16
  const definitions = [];
16
17
  const add = (result, nodes, module) => {
17
18
  for (const node of nodes)
@@ -100,6 +100,11 @@ export declare function parsePlainSource(plainSourceText: string, codeVariables:
100
100
  export declare function modulePlainFile(moduleName: string, templateDirs: string[]): string | null;
101
101
  export declare function readModulePlainSource(moduleName: string, templateDirs: string[]): string;
102
102
  export declare function parsePlainFile(moduleName: string, codeVariables: Record<string, Record<string, string>>, templateDirs: string[], importedModules: string[], modulesTrace: string[]): PlainFileParseResult;
103
+ /**
104
+ * A module's file parsed on its own - as `parsePlainFile(moduleName, {}, templateDirs, [], [])` parses it - shared with
105
+ * every caller, not a copy: for what only reads it. What builds on a parse asks `parsePlainFile` for its own copy.
106
+ */
107
+ export declare function sharedPlainFileParse(moduleName: string, templateDirs: string[]): PlainFileParseResult;
103
108
  /**
104
109
  * Parse every required module, deepest first, without validating it - each with code variables of its own - and
105
110
  * return the definitions the directly required modules export. A module listed again right after it was walked is
@@ -151,8 +156,8 @@ export declare function parseModuleChain(plainSourceFileName: string, templateDi
151
156
  *
152
157
  * Walking a requires chain parses each module in it, and each parse gathers what it inherits
153
158
  * by parsing the modules below - so looking across a project of long chains asks for the
154
- * same files thousands of times. Inside `run` they are parsed once. Outside it nothing is
155
- * kept, so an edit made between two calls is always seen: the cache is for one question
156
- * about the specs, not for the life of the process.
159
+ * same files thousands of times. Inside `run` they are parsed once. Between calls only
160
+ * parses that still hold are kept - a top-level parse by its fingerprint, a file parsed on
161
+ * its own by every file it read - so an edit made between two calls is always seen.
157
162
  */
158
163
  export declare function withParseCache<T>(run: () => T): T;
@@ -4,7 +4,7 @@ import { extname, isAbsolute, basename, join, resolve } from "node:path";
4
4
  import { load as loadYaml } from "js-yaml";
5
5
  import { extractConceptsFromDefinition, findConceptDefinitionsInPlainSource, sortDefinitions, validateConcepts, DEFAULT_CONCEPTS, } from "./concepts.js";
6
6
  import { ImportedModuleWithFunctionalitiesError, MissingFunctionalitiesError, ModuleDoesNotExistError, PlainSyntaxError, UnsupportedBase64Content, } from "./errors.js";
7
- import { findIn, findLargeBase64Blob, getLoadedTemplates, openFrom } from "./files.js";
7
+ import { addReads, findIn, findLargeBase64Blob, getLoadedTemplates, noteReads, openFrom, readsHold } from "./files.js";
8
8
  import { renderPlainSource } from "./liquid.js";
9
9
  import { isStdlibModule, readStdlibModule, stdlibModuleFile } from "./stdlib.js";
10
10
  import { getRawText, lineNumber, parseMarkdown, removeQuotes, renderToMarkdown, renderTokens, traverse, } from "./markdown.js";
@@ -547,6 +547,45 @@ export function readModulePlainSource(moduleName, templateDirs) {
547
547
  return plainSourceText;
548
548
  }
549
549
  export function parsePlainFile(moduleName, codeVariables, templateDirs, importedModules, modulesTrace) {
550
+ // A file parsed on its own - not an import inside another's parse, whose result hangs on what was imported before it -
551
+ // is parsed once while nothing it read has changed: every module below a module in its chain parses each of them
552
+ // again, and a project of long chains asked the same files thousands of times.
553
+ const own = importedModules.length === 0 && modulesTrace.length === 0 && Object.keys(codeVariables).length === 0;
554
+ if (!own)
555
+ return parsePlainFileNow(moduleName, codeVariables, templateDirs, importedModules, modulesTrace);
556
+ // A copy: the caller builds on what it is given.
557
+ const copy = structuredClone(keptParse(moduleName, templateDirs));
558
+ Object.assign(codeVariables, copy.codeVariables);
559
+ return copy.result;
560
+ }
561
+ /** Files parsed on their own, with what each read - by working folder, template folders and module. Oldest out first. */
562
+ const keptFiles = new Map();
563
+ const KEPT_FILES = 500;
564
+ /**
565
+ * A module's file parsed on its own - as `parsePlainFile(moduleName, {}, templateDirs, [], [])` parses it - shared with
566
+ * every caller, not a copy: for what only reads it. What builds on a parse asks `parsePlainFile` for its own copy.
567
+ */
568
+ export function sharedPlainFileParse(moduleName, templateDirs) {
569
+ return keptParse(moduleName, templateDirs).result;
570
+ }
571
+ /** A file parsed on its own, as kept while what it read holds - not to be changed: copy what is built on. */
572
+ function keptParse(moduleName, templateDirs) {
573
+ const key = `${process.cwd()}\0${templateDirs.join("\0")}\0${moduleName}`;
574
+ const kept = keptFiles.get(key);
575
+ if (kept !== undefined && readsHold(kept.reads)) {
576
+ addReads(undefined, kept.reads);
577
+ return kept.parsed;
578
+ }
579
+ const codeVariables = {};
580
+ const { value: result, reads } = noteReads(() => parsePlainFileNow(moduleName, codeVariables, templateDirs, [], []));
581
+ const parsed = { result, codeVariables };
582
+ keptFiles.delete(key);
583
+ keptFiles.set(key, { reads, parsed });
584
+ while (keptFiles.size > KEPT_FILES)
585
+ keptFiles.delete(keptFiles.keys().next().value);
586
+ return parsed;
587
+ }
588
+ function parsePlainFileNow(moduleName, codeVariables, templateDirs, importedModules, modulesTrace) {
550
589
  const plainSourceText = readModulePlainSource(moduleName, templateDirs);
551
590
  return parsePlainSource(plainSourceText, codeVariables, templateDirs, importedModules, [...modulesTrace, moduleName], moduleName);
552
591
  }
@@ -567,7 +606,9 @@ export function processRequiredModules(requiredModules, templateDirs, allRequire
567
606
  exportedDefinitions.push(...(exports.get(moduleName) ?? []));
568
607
  continue;
569
608
  }
570
- const result = parsePlainFile(moduleName, {}, templateDirs, [], []);
609
+ // Only what it requires and what it exports are read: the kept parse, not a copy - but for what it exports, which the
610
+ // module requiring it takes into its own definitions.
611
+ const { result } = keptParse(moduleName, templateDirs);
571
612
  if (result.requiredModules.length === 0) {
572
613
  if (allRequiredModules.length > 0) {
573
614
  // Required modules must form one chain; independent modules cannot yet be merged.
@@ -577,7 +618,7 @@ export function processRequiredModules(requiredModules, templateDirs, allRequire
577
618
  else {
578
619
  processRequiredModules(result.requiredModules, templateDirs, allRequiredModules, [...modulesTrace, moduleName], exports);
579
620
  }
580
- const own = exportedDefinitionsOf(result);
621
+ const own = structuredClone(exportedDefinitionsOf(result));
581
622
  exports.set(moduleName, own);
582
623
  exportedDefinitions.push(...own);
583
624
  allRequiredModules.push(moduleName);
@@ -687,9 +728,9 @@ export function parseModuleChain(plainSourceFileName, templateDirs, options = {}
687
728
  *
688
729
  * Walking a requires chain parses each module in it, and each parse gathers what it inherits
689
730
  * by parsing the modules below - so looking across a project of long chains asks for the
690
- * same files thousands of times. Inside `run` they are parsed once. Outside it nothing is
691
- * kept, so an edit made between two calls is always seen: the cache is for one question
692
- * about the specs, not for the life of the process.
731
+ * same files thousands of times. Inside `run` they are parsed once. Between calls only
732
+ * parses that still hold are kept - a top-level parse by its fingerprint, a file parsed on
733
+ * its own by every file it read - so an edit made between two calls is always seen.
693
734
  */
694
735
  export function withParseCache(run) {
695
736
  if (activeCache !== undefined)
@@ -1,8 +1,8 @@
1
- import { existsSync, realpathSync } from "node:fs";
1
+ import { realpathSync } from "node:fs";
2
2
  import { dirname, join } from "node:path";
3
3
  import { fileURLToPath } from "node:url";
4
4
  import { ModuleDoesNotExistError, PlainSyntaxError } from "./errors.js";
5
- import { findIn, openFrom } from "./files.js";
5
+ import { findIn, isFile, openFrom } from "./files.js";
6
6
  /**
7
7
  * The ***plain standard library's names. `stdlib/` is a reserved prefix: `stdlib/backend/python-fastapi`
8
8
  * is always turri's own template, in its `stdlib/` folder - never a module of the project's. A
@@ -36,7 +36,7 @@ export function readStdlibModule(moduleName, templateDirs) {
36
36
  const own = join(STDLIB_ROOT, file);
37
37
  for (const dir of templateDirs) {
38
38
  const local = join(dir, file);
39
- if (existsSync(local) && !sameFile(local, own)) {
39
+ if (isFile(local) && !sameFile(local, own)) {
40
40
  throw new PlainSyntaxError(`Plain syntax error: '${name}' is a reserved name - modules under ${STDLIB_PREFIX} are turri's standard library - ` +
41
41
  `but ${local} is a module of that name. Rename it, or the folder it is in.`);
42
42
  }
@@ -31,6 +31,8 @@ export declare function baseFolderOf(plainFile: string): {
31
31
  * does not ignore - so no dependencies or build output; otherwise every file but those of the folders never copied.
32
32
  */
33
33
  export declare function baseFiles(folder: string): string[];
34
+ /** `baseFiles`, its git runs not holding the thread up - for doctor, on a host's thread. */
35
+ export declare function baseFilesAsync(folder: string): Promise<string[]>;
34
36
  /** What a base folder holds now, as one hash: each file's path and bytes. */
35
37
  export declare function hashBase(folder: string, files?: string[]): string;
36
38
  /** What a codebase started from, when it started from a base folder. */
@@ -1,4 +1,5 @@
1
1
  import { spawnSync } from "node:child_process";
2
+ import { spawnAsync } from "../spawn-async.js";
2
3
  import { createHash } from "node:crypto";
3
4
  import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
4
5
  import { dirname, join, relative, resolve, sep } from "node:path";
@@ -34,14 +35,31 @@ const NOT_COPIED = new Set([".git", "node_modules", ".venv", "venv", "__pycache_
34
35
  * does not ignore - so no dependencies or build output; otherwise every file but those of the folders never copied.
35
36
  */
36
37
  export function baseFiles(folder) {
37
- const inGit = spawnSync("git", ["-C", folder, "rev-parse", "--is-inside-work-tree"], { encoding: "utf8" });
38
+ const inGit = spawnSync("git", ["-C", folder, ...IN_WORK_TREE], { encoding: "utf8" });
38
39
  if (inGit.status === 0 && inGit.stdout.trim() === "true") {
39
- const listed = spawnSync("git", ["-C", folder, "ls-files", "-z", "--cached", "--others", "--exclude-standard", "--", "."], { encoding: "utf8", maxBuffer: 256 * 1024 * 1024 });
40
- if (listed.status === 0) {
41
- // A tracked file deleted in the work tree is listed, and not there to copy.
42
- return [...new Set(listed.stdout.split("\0").filter((path) => path !== ""))].filter((path) => isFile(join(folder, path))).sort();
43
- }
40
+ const listed = spawnSync("git", ["-C", folder, ...LISTED], { encoding: "utf8", maxBuffer: 256 * 1024 * 1024 });
41
+ if (listed.status === 0)
42
+ return gitListed(folder, listed.stdout);
43
+ }
44
+ return walkedFiles(folder);
45
+ }
46
+ /** `baseFiles`, its git runs not holding the thread up - for doctor, on a host's thread. */
47
+ export async function baseFilesAsync(folder) {
48
+ const inGit = await spawnAsync("git", ["-C", folder, ...IN_WORK_TREE]);
49
+ if (inGit.status === 0 && inGit.stdout.trim() === "true") {
50
+ const listed = await spawnAsync("git", ["-C", folder, ...LISTED]);
51
+ if (listed.status === 0)
52
+ return gitListed(folder, listed.stdout);
44
53
  }
54
+ return walkedFiles(folder);
55
+ }
56
+ const IN_WORK_TREE = ["rev-parse", "--is-inside-work-tree"];
57
+ const LISTED = ["ls-files", "-z", "--cached", "--others", "--exclude-standard", "--", "."];
58
+ /** What git listed, as files there: a tracked file deleted in the work tree is listed, and not there to copy. */
59
+ function gitListed(folder, listed) {
60
+ return [...new Set(listed.split("\0").filter((path) => path !== ""))].filter((path) => isFile(join(folder, path))).sort();
61
+ }
62
+ function walkedFiles(folder) {
45
63
  const found = [];
46
64
  const walk = (dir) => {
47
65
  for (const entry of readdirSync(dir)) {
@@ -26,7 +26,7 @@ export interface CommandFence {
26
26
  /** Whose paths these are: drive letters, backslashes and any case on Windows. */
27
27
  platform?: "posix" | "win32";
28
28
  }
29
- export type CheckRule = "path" | "record" | "credential" | "install" | "kill";
29
+ export type CheckRule = "path" | "record" | "credential" | "install" | "system" | "kill";
30
30
  export type CheckResult = {
31
31
  ok: true;
32
32
  } | {
@@ -36,5 +36,16 @@ export type CheckResult = {
36
36
  };
37
37
  /** What of the machine an agent's command may read, beside its roots and its toolchains: the system's own folders. */
38
38
  export declare function machineReadable(platform?: NodeJS.Platform): string[];
39
+ /**
40
+ * Why a command line changes the machine - installs software outside the project, or changes a setting of the
41
+ * machine's or the person's environment - or nothing when it does not. The project's own dependencies, installed into
42
+ * the project (npm install, a venv, mvn, cargo), change nothing of the machine's. turri changes the machine nowhere: not
43
+ * an agent, not the chat, not a release - what the machine lacks is the person's to install, in their own words.
44
+ */
45
+ export declare function machineChange(command: string): string | undefined;
46
+ /** The .plain file a command line writes - a ***plain module or a test plain file - or nothing when it writes none. */
47
+ export declare function writesPlain(command: string): string | undefined;
48
+ /** What turri says when a command would change the machine: what it does instead. */
49
+ export declare function machineChangeReason(why: string): string;
39
50
  /** Check one command line against a role's fence: `{ ok: true }`, or which rule it breaks and what to do instead. */
40
51
  export declare function checkCommand(command: string, fence: CommandFence): CheckResult;