kibi-cli 2.39.0 → 2.40.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. package/dist/cli-register-foundation.js +2 -2
  2. package/dist/commands/check.d.ts.map +1 -1
  3. package/dist/commands/check.js +33 -9
  4. package/dist/commands/doctor.d.ts.map +1 -1
  5. package/dist/commands/doctor.js +15 -0
  6. package/dist/commands/init.d.ts.map +1 -1
  7. package/dist/commands/init.js +4 -0
  8. package/dist/commands/migrate-plan-record.d.ts +18 -0
  9. package/dist/commands/migrate-plan-record.d.ts.map +1 -0
  10. package/dist/commands/migrate-plan-record.js +112 -0
  11. package/dist/commands/migrate.d.ts +11 -2
  12. package/dist/commands/migrate.d.ts.map +1 -1
  13. package/dist/commands/migrate.js +53 -7
  14. package/dist/commands/prove.d.ts.map +1 -1
  15. package/dist/commands/prove.js +18 -3
  16. package/dist/commands/sync/manifest.d.ts.map +1 -1
  17. package/dist/commands/sync/manifest.js +17 -4
  18. package/dist/commands/sync.d.ts.map +1 -1
  19. package/dist/commands/sync.js +8 -0
  20. package/dist/commands/telemetry.d.ts.map +1 -1
  21. package/dist/commands/telemetry.js +12 -2
  22. package/dist/extractors/symbol-title-match.d.ts +31 -0
  23. package/dist/extractors/symbol-title-match.d.ts.map +1 -0
  24. package/dist/extractors/symbol-title-match.js +133 -0
  25. package/dist/extractors/symbols-coordinator.d.ts.map +1 -1
  26. package/dist/extractors/symbols-coordinator.js +81 -12
  27. package/dist/operations/bootstrap/discovery-evidence.js +2 -2
  28. package/dist/operations/bootstrap/guidance-actions.d.ts.map +1 -1
  29. package/dist/operations/bootstrap/guidance-actions.js +12 -1
  30. package/dist/operations/bootstrap/plugin-offers.d.ts +7 -1
  31. package/dist/operations/bootstrap/plugin-offers.d.ts.map +1 -1
  32. package/dist/operations/bootstrap/plugin-offers.js +73 -3
  33. package/dist/operations/modeling/logic-modeling.d.ts.map +1 -1
  34. package/dist/operations/modeling/logic-modeling.js +2 -1
  35. package/dist/operations/modeling/model-requirement.d.ts.map +1 -1
  36. package/dist/operations/modeling/model-requirement.js +2 -1
  37. package/dist/operations/mutation/symbol-refresh.d.ts +2 -0
  38. package/dist/operations/mutation/symbol-refresh.d.ts.map +1 -1
  39. package/dist/operations/mutation/symbol-refresh.js +6 -1
  40. package/dist/operations/mutation/upsert.d.ts.map +1 -1
  41. package/dist/operations/mutation/upsert.js +7 -0
  42. package/dist/operations/planning/compile-intent.d.ts.map +1 -1
  43. package/dist/operations/planning/compile-intent.js +2 -1
  44. package/dist/operations/proof/claim-bindings.d.ts +11 -0
  45. package/dist/operations/proof/claim-bindings.d.ts.map +1 -1
  46. package/dist/operations/proof/claim-bindings.js +144 -1
  47. package/dist/operations/proof/code-certificates.d.ts +10 -3
  48. package/dist/operations/proof/code-certificates.d.ts.map +1 -1
  49. package/dist/operations/proof/code-certificates.js +65 -13
  50. package/dist/operations/proof/code-languages.d.ts +104 -0
  51. package/dist/operations/proof/code-languages.d.ts.map +1 -0
  52. package/dist/operations/proof/code-languages.js +119 -0
  53. package/dist/operations/proof/code-models.d.ts +2 -0
  54. package/dist/operations/proof/code-models.d.ts.map +1 -1
  55. package/dist/operations/proof/code-models.js +11 -5
  56. package/dist/operations/proof/code-runner.d.ts +12 -9
  57. package/dist/operations/proof/code-runner.d.ts.map +1 -1
  58. package/dist/operations/proof/code-runner.js +81 -38
  59. package/dist/operations/proof/code-scope.d.ts +8 -0
  60. package/dist/operations/proof/code-scope.d.ts.map +1 -1
  61. package/dist/operations/proof/code-scope.js +30 -1
  62. package/dist/operations/proof/explain.d.ts +13 -0
  63. package/dist/operations/proof/explain.d.ts.map +1 -1
  64. package/dist/operations/proof/explain.js +66 -0
  65. package/dist/operations/proof/ingest-proof.d.ts +2 -0
  66. package/dist/operations/proof/ingest-proof.d.ts.map +1 -1
  67. package/dist/operations/proof/ingest-proof.js +1 -0
  68. package/dist/operations/proof/java-package-shadow.d.ts +14 -0
  69. package/dist/operations/proof/java-package-shadow.d.ts.map +1 -0
  70. package/dist/operations/proof/java-package-shadow.js +389 -0
  71. package/dist/operations/proof/prolog-evaluator.d.ts +14 -0
  72. package/dist/operations/proof/prolog-evaluator.d.ts.map +1 -0
  73. package/dist/operations/proof/prolog-evaluator.js +247 -0
  74. package/dist/operations/proof/prolog-source-reader.d.ts +20 -0
  75. package/dist/operations/proof/prolog-source-reader.d.ts.map +1 -0
  76. package/dist/operations/proof/prolog-source-reader.js +1123 -0
  77. package/dist/operations/proof/source-code-readers.d.ts +6 -0
  78. package/dist/operations/proof/source-code-readers.d.ts.map +1 -1
  79. package/dist/operations/proof/source-code-readers.js +879 -75
  80. package/dist/plugins/approved-source-analyzers.generated.d.ts +13 -35
  81. package/dist/plugins/approved-source-analyzers.generated.d.ts.map +1 -1
  82. package/dist/plugins/approved-source-analyzers.generated.js +13 -35
  83. package/dist/plugins/plugin-install-check.d.ts +21 -0
  84. package/dist/plugins/plugin-install-check.d.ts.map +1 -0
  85. package/dist/plugins/plugin-install-check.js +62 -0
  86. package/dist/plugins/proof-reference-extractors.d.ts +20 -0
  87. package/dist/plugins/proof-reference-extractors.d.ts.map +1 -0
  88. package/dist/plugins/proof-reference-extractors.js +104 -0
  89. package/dist/plugins/reference-language-suggestion.d.ts +51 -0
  90. package/dist/plugins/reference-language-suggestion.d.ts.map +1 -0
  91. package/dist/plugins/reference-language-suggestion.js +277 -0
  92. package/dist/plugins/registry.d.ts.map +1 -1
  93. package/dist/plugins/registry.js +3 -1
  94. package/dist/plugins/source-analysis-service.d.ts.map +1 -1
  95. package/dist/plugins/source-analysis-service.js +4 -0
  96. package/dist/plugins/source-change-analysis.d.ts +26 -1
  97. package/dist/plugins/source-change-analysis.d.ts.map +1 -1
  98. package/dist/plugins/source-change-analysis.js +77 -5
  99. package/dist/plugins/source-classification.d.ts.map +1 -1
  100. package/dist/plugins/source-classification.js +23 -3
  101. package/dist/public/operations/catalog.d.ts +32 -7
  102. package/dist/public/operations/catalog.d.ts.map +1 -1
  103. package/dist/public/operations/check-executor.d.ts.map +1 -1
  104. package/dist/public/operations/check-executor.js +3 -0
  105. package/dist/public/operations/evidence-floor.d.ts +8 -0
  106. package/dist/public/operations/evidence-floor.d.ts.map +1 -1
  107. package/dist/public/operations/evidence-floor.js +19 -1
  108. package/dist/public/operations/specs/check-rules.generated.d.ts +1 -1
  109. package/dist/public/operations/specs/check-rules.generated.d.ts.map +1 -1
  110. package/dist/public/operations/specs/check-rules.generated.js +2 -1
  111. package/dist/public/operations/specs/check.d.ts +1 -1
  112. package/dist/public/operations/specs/mutation-schemas.d.ts +31 -6
  113. package/dist/public/operations/specs/mutation-schemas.d.ts.map +1 -1
  114. package/dist/public/operations/specs/reporting.d.ts +8 -0
  115. package/dist/public/operations/specs/reporting.d.ts.map +1 -1
  116. package/dist/public/operations/specs/reporting.js +26 -3
  117. package/dist/public/operations/specs/semantic.d.ts.map +1 -1
  118. package/dist/public/operations/specs/semantic.js +2 -1
  119. package/dist/public/operations/specs/upsert.d.ts +31 -6
  120. package/dist/public/operations/specs/upsert.d.ts.map +1 -1
  121. package/dist/public/operations/symbol-coordinates-check.d.ts +11 -0
  122. package/dist/public/operations/symbol-coordinates-check.d.ts.map +1 -0
  123. package/dist/public/operations/symbol-coordinates-check.js +66 -0
  124. package/dist/public/proof-protocol.d.ts +31 -6
  125. package/dist/public/proof-protocol.d.ts.map +1 -1
  126. package/dist/public/proof-protocol.js +45 -15
  127. package/dist/public/proof-sources.d.ts +4 -2
  128. package/dist/public/proof-sources.d.ts.map +1 -1
  129. package/dist/public/proof-sources.js +43 -4
  130. package/dist/public/skills/kibi-usage/SKILL.md +6 -3
  131. package/dist/public/skills/kibi-usage/resources/prolog-proofs.md +207 -0
  132. package/dist/public/skills/kibi-usage/resources/proof.md +44 -3
  133. package/dist/schemas/entity.schema.json +34 -6
  134. package/dist/traceability/analysis-gate.d.ts +3 -1
  135. package/dist/traceability/analysis-gate.d.ts.map +1 -1
  136. package/dist/traceability/analysis-gate.js +9 -1
  137. package/dist/traceability/staged-file-coverage.d.ts.map +1 -1
  138. package/dist/traceability/staged-file-coverage.js +7 -2
  139. package/dist/traceability/symbol-extract.d.ts +17 -0
  140. package/dist/traceability/symbol-extract.d.ts.map +1 -1
  141. package/dist/traceability/symbol-extract.js +69 -15
  142. package/dist/traceability/validate.d.ts.map +1 -1
  143. package/dist/traceability/validate.js +5 -7
  144. package/dist/utils/first-sentence.d.ts +9 -0
  145. package/dist/utils/first-sentence.d.ts.map +1 -0
  146. package/dist/utils/first-sentence.js +52 -0
  147. package/dist/utils/rule-registry.generated.d.ts +6 -0
  148. package/dist/utils/rule-registry.generated.d.ts.map +1 -1
  149. package/dist/utils/rule-registry.generated.js +7 -0
  150. package/package.json +5 -5
  151. package/src/public/operations/check-executor.ts +8 -0
  152. package/src/public/operations/evidence-floor.ts +29 -1
  153. package/src/public/operations/specs/check-rules.generated.ts +2 -1
  154. package/src/public/operations/specs/reporting.ts +38 -3
  155. package/src/public/operations/specs/semantic.ts +2 -1
  156. package/src/public/operations/symbol-coordinates-check.ts +79 -0
  157. package/src/public/proof-protocol-types.d.ts +22 -5
  158. package/src/public/proof-protocol.ts +78 -19
  159. package/src/public/proof-sources.ts +51 -3
  160. package/src/public/skills/kibi-usage/SKILL.md +6 -3
  161. package/src/public/skills/kibi-usage/resources/prolog-proofs.md +207 -0
  162. package/src/public/skills/kibi-usage/resources/proof.md +44 -3
  163. package/src/schemas/entity.schema.json +34 -6
@@ -19,16 +19,24 @@
19
19
  * The readers of `source` specs that need code: the references reader
20
20
  * (environment variables, imports, calls, string arguments, thrown errors
21
21
  * and property values, read from the syntax tree of TypeScript, JavaScript
22
- * and Python files), and the readers that load a module (drizzle schemas,
22
+ * and Python files, from the terms of SWI-Prolog files, and from the syntax
23
+ * trees of Go, Java, C# and Rust files through a qualified
24
+ * kibi.reference-extractor.v1 plugin), the prolog
25
+ * reader (fact arguments of a Prolog file), and the readers that load a module (drizzle schemas,
23
26
  * Hono and Express route tables, zod and pydantic validators).
24
27
  */
28
+ import { execFileSync } from "node:child_process";
25
29
  import { readFileSync } from "node:fs";
26
- import { createRequire } from "node:module";
30
+ import { builtinModules, createRequire } from "node:module";
27
31
  import path from "node:path";
28
32
  import { matchWorkspaceFiles, unresolvedReason, } from "../../public/proof-sources.js";
33
+ import { validateReferenceExtractionResult } from "kibi-plugin-sdk";
34
+ import { REFERENCE_PLUGIN_PACKAGE, proofReferenceExtractorLoader, } from "../../plugins/proof-reference-extractors.js";
35
+ import { codeLanguage, pluginReferenceLanguage, unsupportedLanguageReason } from "./code-languages.js";
36
+ import { createShadowRowFilter } from "./java-package-shadow.js";
29
37
  import { runCodeEvaluator, TS_RESOLVE_HOOKS } from "./code-runner.js";
30
- const JS_FILE = /\.(?:[cm]?[jt]sx?)$/i;
31
- const PY_FILE = /\.py$/i;
38
+ import { PROLOG_SOURCE_READER } from "./prolog-source-reader.js";
39
+ const NODE_BUILTINS = new Set(builtinModules.filter((name) => !name.startsWith("_")));
32
40
  function globList(value) {
33
41
  if (typeof value === "string")
34
42
  return [value];
@@ -76,16 +84,20 @@ export function extractScriptReferences(ts, file, text, spec) {
76
84
  const literal = (node) => node !== undefined && (ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node))
77
85
  ? node.text
78
86
  : null;
79
- // The expression under parentheses, `!`, `as T`, `<T>x` and `satisfies T`:
80
- // a cast does not change what is read, so `(import.meta as X).env` is
81
- // import.meta.env.
87
+ // The expression under parentheses, `!`, `as T`, `<T>x`, `satisfies T`,
88
+ // `await` and the comma operator: none of them changes what is read, so
89
+ // `(import.meta as X).env` is import.meta.env and `(0, process).env` is
90
+ // process.env.
82
91
  const unwrapped = (node) => {
83
92
  if (ts.isNonNullExpression(node) ||
84
93
  ts.isParenthesizedExpression(node) ||
85
94
  ts.isAsExpression(node) ||
86
95
  ts.isTypeAssertionExpression(node) ||
87
- ts.isSatisfiesExpression(node))
96
+ ts.isSatisfiesExpression(node) ||
97
+ ts.isAwaitExpression(node))
88
98
  return unwrapped(node.expression);
99
+ if (ts.isBinaryExpression(node) && node.operatorToken.kind === ts.SyntaxKind.CommaToken)
100
+ return unwrapped(node.right);
89
101
  return node;
90
102
  };
91
103
  // a.b.c for identifier and property chains, a["b"] for literal element access
@@ -120,10 +132,217 @@ export function extractScriptReferences(ts, file, text, spec) {
120
132
  (ts.isPropertyAccessExpression(parent) || ts.isElementAccessExpression(parent)) &&
121
133
  parent.expression === current);
122
134
  };
135
+ // The node an expression stands in, above its casts and parentheses.
136
+ const outer = (node) => {
137
+ let current = node;
138
+ while (current.parent !== undefined && unwrapped(current.parent) !== current.parent)
139
+ current = current.parent;
140
+ return current;
141
+ };
123
142
  const extract = spec.extract;
124
143
  const callee = typeof spec.callee === "string" ? spec.callee : "";
125
144
  const key = typeof spec.key === "string" ? spec.key : "";
126
145
  const ENV_OBJECTS = new Set(["process.env", "import.meta.env", "Bun.env"]);
146
+ // A Node built-in under its `node:` name is the same module.
147
+ const moduleName = (specifier) => specifier.startsWith("node:") && NODE_BUILTINS.has(specifier.slice(5)) ? specifier.slice(5) : specifier;
148
+ const spellings = (specifier) => [...new Set([specifier, moduleName(specifier)])];
149
+ // What each local name stands for: `import { execSync as run } from
150
+ // "child_process"` makes run child_process.execSync, `const { env } =
151
+ // process` makes env process.env, `const g = globalThis` makes g
152
+ // globalThis. Names are file-wide; a shadowing local is read as the import,
153
+ // which can list more than the code does but never less.
154
+ const names = new Map();
155
+ // Local names bound to a module itself (import * as cp, const cp = require(...)).
156
+ const moduleNames = new Set();
157
+ // The declarations those names come from.
158
+ const bindingDeclarations = new Set();
159
+ const GLOBAL_OBJECTS = new Set(["globalThis", "global", "window", "self"]);
160
+ const canonical = (written) => {
161
+ let current = written;
162
+ for (let depth = 0; depth < 8; depth++) {
163
+ const dot = current.indexOf(".");
164
+ const head = dot < 0 ? current : current.slice(0, dot);
165
+ const bound = names.get(head);
166
+ if (bound === undefined || bound === head)
167
+ break;
168
+ current = bound + (dot < 0 ? "" : current.slice(dot));
169
+ }
170
+ // globalThis.process is process.
171
+ const parts = current.split(".");
172
+ while (parts.length > 1 && GLOBAL_OBJECTS.has(parts[0] ?? "") && !names.has(parts[0] ?? ""))
173
+ parts.shift();
174
+ return parts.join(".");
175
+ };
176
+ const LOADERS = new Set(["require", "module.require", "process.getBuiltinModule"]);
177
+ const REQUIRE_FACTORIES = new Set(["createRequire", "module.createRequire"]);
178
+ // A call that loads a module: require("m"), import("m"),
179
+ // process.getBuiltinModule("m"), createRequire(url)("m"). Undefined when the
180
+ // call loads nothing; `{ specifier: null }` when the module is not a literal.
181
+ const loaded = (node) => {
182
+ const call = unwrapped(node);
183
+ if (!ts.isCallExpression(call))
184
+ return undefined;
185
+ const target = unwrapped(call.expression);
186
+ const isLoader = call.expression.kind === ts.SyntaxKind.ImportKeyword ||
187
+ LOADERS.has(canonical(chain(target) ?? "")) ||
188
+ (ts.isCallExpression(target) && REQUIRE_FACTORIES.has(canonical(chain(target.expression) ?? "")));
189
+ if (!isLoader)
190
+ return undefined;
191
+ const value = literal(call.arguments[0]);
192
+ return { specifier: value === null ? null : moduleName(value) };
193
+ };
194
+ const bindPattern = (pattern, from) => {
195
+ if (ts.isIdentifier(pattern)) {
196
+ names.set(pattern.text, from);
197
+ return;
198
+ }
199
+ if (!ts.isObjectBindingPattern(pattern))
200
+ return;
201
+ for (const element of pattern.elements) {
202
+ if (element.dotDotDotToken)
203
+ continue;
204
+ const property = element.propertyName ?? element.name;
205
+ if (ts.isIdentifier(property) || ts.isStringLiteral(property))
206
+ bindPattern(element.name, `${from}.${property.text}`);
207
+ }
208
+ };
209
+ const collect = (node) => {
210
+ if (ts.isImportDeclaration(node) && node.importClause && !node.importClause.isTypeOnly) {
211
+ const from = literal(node.moduleSpecifier);
212
+ if (from !== null) {
213
+ const mod = moduleName(from);
214
+ const clause = node.importClause;
215
+ if (clause.name) {
216
+ names.set(clause.name.text, mod);
217
+ moduleNames.add(clause.name.text);
218
+ }
219
+ const bindings = clause.namedBindings;
220
+ if (bindings && ts.isNamespaceImport(bindings)) {
221
+ names.set(bindings.name.text, mod);
222
+ moduleNames.add(bindings.name.text);
223
+ }
224
+ else if (bindings && ts.isNamedImports(bindings)) {
225
+ for (const element of bindings.elements) {
226
+ if (element.isTypeOnly)
227
+ continue;
228
+ const imported = (element.propertyName ?? element.name).text;
229
+ names.set(element.name.text, imported === "default" ? mod : `${mod}.${imported}`);
230
+ if (imported === "default")
231
+ moduleNames.add(element.name.text);
232
+ }
233
+ }
234
+ }
235
+ }
236
+ else if (ts.isImportEqualsDeclaration(node) && ts.isExternalModuleReference(node.moduleReference)) {
237
+ const from = literal(node.moduleReference.expression);
238
+ if (from !== null) {
239
+ names.set(node.name.text, moduleName(from));
240
+ moduleNames.add(node.name.text);
241
+ }
242
+ }
243
+ else if (ts.isVariableDeclaration(node) && node.initializer) {
244
+ bindingDeclarations.add(node);
245
+ const init = unwrapped(node.initializer);
246
+ const module = loaded(init);
247
+ if (module !== undefined) {
248
+ if (module.specifier !== null) {
249
+ bindPattern(node.name, module.specifier);
250
+ if (ts.isIdentifier(node.name))
251
+ moduleNames.add(node.name.text);
252
+ }
253
+ }
254
+ else if (ts.isCallExpression(init) && REQUIRE_FACTORIES.has(canonical(chain(init.expression) ?? ""))) {
255
+ // const req = createRequire(import.meta.url): req loads modules.
256
+ if (ts.isIdentifier(node.name))
257
+ names.set(node.name.text, "require");
258
+ }
259
+ else {
260
+ const from = chain(init);
261
+ if (from !== null)
262
+ bindPattern(node.name, from);
263
+ }
264
+ }
265
+ ts.forEachChild(node, collect);
266
+ };
267
+ collect(source);
268
+ // Bound names are read through what they stand for; a declaration's own
269
+ // name is not a read.
270
+ const declares = (node) => {
271
+ const parent = node.parent;
272
+ if (parent === undefined)
273
+ return false;
274
+ return (((ts.isVariableDeclaration(parent) ||
275
+ ts.isBindingElement(parent) ||
276
+ ts.isParameter(parent) ||
277
+ ts.isFunctionDeclaration(parent) ||
278
+ ts.isClassDeclaration(parent) ||
279
+ ts.isImportClause(parent) ||
280
+ ts.isNamespaceImport(parent) ||
281
+ ts.isImportSpecifier(parent) ||
282
+ ts.isImportEqualsDeclaration(parent) ||
283
+ ts.isExportSpecifier(parent) ||
284
+ ts.isPropertyAssignment(parent) ||
285
+ ts.isPropertyDeclaration(parent) ||
286
+ ts.isMethodDeclaration(parent) ||
287
+ ts.isPropertySignature(parent) ||
288
+ ts.isLabeledStatement(parent)) &&
289
+ parent.name === node) ||
290
+ (ts.isBindingElement(parent) && parent.propertyName === node) ||
291
+ (ts.isImportSpecifier(parent) && parent.propertyName === node) ||
292
+ (ts.isExportSpecifier(parent) && parent.propertyName === node) ||
293
+ (ts.isPropertyAccessExpression(parent) && parent.name === node) ||
294
+ (ts.isQualifiedName(parent) && parent.right === node));
295
+ };
296
+ const inType = (node) => {
297
+ for (let current = node.parent; current !== undefined; current = current.parent) {
298
+ if (ts.isTypeNode(current) || ts.isTypeQueryNode(current) || ts.isHeritageClause(current))
299
+ return true;
300
+ if (ts.isStatement(current) || ts.isSourceFile(current))
301
+ return false;
302
+ }
303
+ return false;
304
+ };
305
+ // Whether a global name (process, globalThis) is a parameter or local of
306
+ // an enclosing function or block there.
307
+ const shadowed = (node, name) => {
308
+ const binds = (binding) => ts.isIdentifier(binding)
309
+ ? binding.text === name
310
+ : binding.elements.some((element) => !ts.isOmittedExpression(element) && binds(element.name));
311
+ for (let current = node.parent; current !== undefined; current = current.parent) {
312
+ if (ts.isFunctionLike(current) && current.parameters.some((parameter) => binds(parameter.name)))
313
+ return true;
314
+ if (ts.isBlock(current) || ts.isSourceFile(current)) {
315
+ for (const statement of current.statements) {
316
+ if (ts.isVariableStatement(statement) && statement.declarationList.declarations.some((d) => !bindingDeclarations.has(d) && binds(d.name)))
317
+ return true;
318
+ if ((ts.isFunctionDeclaration(statement) || ts.isClassDeclaration(statement)) && statement.name?.text === name)
319
+ return true;
320
+ }
321
+ }
322
+ }
323
+ return false;
324
+ };
325
+ // Whether an expression is a whole chain (not the head of a longer one).
326
+ const maximal = (node) => {
327
+ const top = outer(node);
328
+ const parent = top.parent;
329
+ return !(parent !== undefined &&
330
+ (ts.isPropertyAccessExpression(parent) || ts.isElementAccessExpression(parent)) &&
331
+ parent.expression === top);
332
+ };
333
+ const calledAs = (node) => {
334
+ const top = outer(node);
335
+ const parent = top.parent;
336
+ return (parent !== undefined && (ts.isCallExpression(parent) || ts.isNewExpression(parent)) && parent.expression === top);
337
+ };
338
+ // The initializer of a declaration whose names are followed above.
339
+ const followed = (node) => {
340
+ const top = outer(node);
341
+ const parent = top.parent;
342
+ return parent !== undefined && ts.isVariableDeclaration(parent) && parent.initializer === top;
343
+ };
344
+ // Code built from text runs anything.
345
+ const DYNAMIC_CODE = new Set(["eval", "Function", "vm.runInThisContext", "vm.runInNewContext", "vm.runInContext", "vm.compileFunction", "vm.Script", "vm.SourceTextModule"]);
127
346
  const catchNames = [];
128
347
  const visit = (node) => {
129
348
  if (ts.isCatchClause(node) && node.variableDeclaration && ts.isIdentifier(node.variableDeclaration.name)) {
@@ -132,24 +351,62 @@ export function extractScriptReferences(ts, file, text, spec) {
132
351
  catchNames.pop();
133
352
  return;
134
353
  }
354
+ const reads = extract === "env" || extract === "calls" || extract === "imports";
355
+ if (reads && (ts.isCallExpression(node) || ts.isNewExpression(node))) {
356
+ const name = canonical(chain(node.expression) ?? "");
357
+ if (DYNAMIC_CODE.has(name))
358
+ unresolved.push(at(node));
359
+ const module = loaded(node);
360
+ if (module !== undefined && extract !== "imports" && !followed(node)) {
361
+ // A module loaded in place: what is read or called on it is not followed
362
+ // unless it is a known module whose member is called directly.
363
+ const top = outer(node);
364
+ const parent = top.parent;
365
+ const member = parent !== undefined && ts.isPropertyAccessExpression(parent) && parent.expression === top
366
+ ? parent.name.text
367
+ : null;
368
+ const direct = extract === "calls" &&
369
+ module.specifier !== null &&
370
+ member !== null &&
371
+ parent !== undefined &&
372
+ calledAs(parent);
373
+ if (direct)
374
+ members.push(`${module.specifier}.${member}`);
375
+ else if (module.specifier === null || (extract === "env" && module.specifier === "process"))
376
+ unresolved.push(at(node));
377
+ else if (extract === "calls" && NODE_BUILTINS.has(module.specifier))
378
+ // A built-in module used in place: any of its members may be called.
379
+ members.push(`${module.specifier}.*`);
380
+ }
381
+ }
135
382
  switch (extract) {
136
383
  case "env": {
137
- const name = ts.isPropertyAccessExpression(node) || ts.isElementAccessExpression(node) ? chain(node.expression) : null;
138
- if (name !== null && ENV_OBJECTS.has(name)) {
139
- if (ts.isPropertyAccessExpression(node)) {
140
- if (!(name === "import.meta.env" && VITE_BUILTINS.has(node.name.text)))
141
- members.push(node.name.text);
142
- }
143
- else {
144
- const value = literal(node.argumentExpression);
145
- if (value === null)
146
- unresolved.push(at(node));
147
- else
148
- members.push(value);
149
- }
384
+ // `export { env } from "node:process"` hands the environment to the files that import it.
385
+ if (ts.isExportDeclaration(node) &&
386
+ node.moduleSpecifier &&
387
+ moduleName(literal(node.moduleSpecifier) ?? "") === "process") {
388
+ unresolved.push(at(node));
150
389
  return;
151
390
  }
152
- if (ts.isCallExpression(node) && chain(node.expression) === "Deno.env.get") {
391
+ if (ts.isPropertyAccessExpression(node) || ts.isElementAccessExpression(node)) {
392
+ const written = chain(node.expression);
393
+ const name = written === null ? null : canonical(written);
394
+ if (name !== null && ENV_OBJECTS.has(name)) {
395
+ if (ts.isPropertyAccessExpression(node)) {
396
+ if (!(name === "import.meta.env" && VITE_BUILTINS.has(node.name.text)))
397
+ members.push(node.name.text);
398
+ }
399
+ else {
400
+ const value = literal(node.argumentExpression);
401
+ if (value === null)
402
+ unresolved.push(at(node));
403
+ else
404
+ members.push(value);
405
+ }
406
+ return;
407
+ }
408
+ }
409
+ if (ts.isCallExpression(node) && canonical(chain(node.expression) ?? "") === "Deno.env.get") {
153
410
  const value = literal(node.arguments[0]);
154
411
  if (value === null)
155
412
  unresolved.push(at(node));
@@ -157,26 +414,34 @@ export function extractScriptReferences(ts, file, text, spec) {
157
414
  members.push(value);
158
415
  return;
159
416
  }
160
- if (ts.isVariableDeclaration(node) && node.initializer && ts.isObjectBindingPattern(node.name)) {
161
- const from = chain(node.initializer);
162
- if (from !== null && ENV_OBJECTS.has(from)) {
163
- for (const element of node.name.elements) {
164
- const property = element.propertyName ?? element.name;
165
- if (element.dotDotDotToken || !(ts.isIdentifier(property) || ts.isStringLiteral(property)))
166
- unresolved.push(at(element));
167
- else
168
- members.push(property.text);
169
- }
417
+ if (ts.isVariableDeclaration(node) && node.initializer && !ts.isIdentifier(node.name)) {
418
+ const written = chain(unwrapped(node.initializer));
419
+ const from = written === null ? (loaded(node.initializer)?.specifier ?? null) : canonical(written);
420
+ if (from !== null && (ENV_OBJECTS.has(from) || from === "process" || GLOBAL_OBJECTS.has(from))) {
421
+ readPattern(node.name, from);
170
422
  return;
171
423
  }
172
424
  }
173
- // The whole environment object handed elsewhere can be read under any name.
174
- if ((ts.isPropertyAccessExpression(node) || ts.isMetaProperty(node)) && ENV_OBJECTS.has(chain(node) ?? "")) {
175
- unresolved.push(at(node));
176
- return;
425
+ // The environment object, process or the global object handed on can
426
+ // be read under any name.
427
+ if ((ts.isIdentifier(node) || ts.isPropertyAccessExpression(node) || ts.isMetaProperty(node)) &&
428
+ !declares(node) &&
429
+ !inType(node) &&
430
+ maximal(node) &&
431
+ !followed(node)) {
432
+ const name = canonical(chain(node) ?? "");
433
+ if (ts.isIdentifier(node) && shadowed(node, node.text))
434
+ break;
435
+ const handedOn = ENV_OBJECTS.has(name) ||
436
+ (name === "process" && !readsAProperty(node)) ||
437
+ (GLOBAL_OBJECTS.has(name) && !readsAProperty(node));
438
+ if (handedOn) {
439
+ unresolved.push(at(node));
440
+ return;
441
+ }
177
442
  }
178
443
  // So can import.meta itself, when it is handed on instead of read.
179
- if (ts.isMetaProperty(node) && chain(node) === "import.meta" && !readsAProperty(node)) {
444
+ if (ts.isMetaProperty(node) && chain(node) === "import.meta" && !readsAProperty(node) && !followed(node)) {
180
445
  unresolved.push(at(node));
181
446
  return;
182
447
  }
@@ -186,41 +451,87 @@ export function extractScriptReferences(ts, file, text, spec) {
186
451
  if ((ts.isImportDeclaration(node) || ts.isExportDeclaration(node)) && node.moduleSpecifier) {
187
452
  const value = literal(node.moduleSpecifier);
188
453
  if (value !== null)
189
- members.push(value);
454
+ members.push(...spellings(value));
190
455
  break;
191
456
  }
192
457
  if (ts.isImportEqualsDeclaration(node) && ts.isExternalModuleReference(node.moduleReference)) {
193
458
  const value = literal(node.moduleReference.expression);
194
459
  if (value !== null)
195
- members.push(value);
460
+ members.push(...spellings(value));
196
461
  break;
197
462
  }
198
- if (ts.isCallExpression(node) &&
199
- (node.expression.kind === ts.SyntaxKind.ImportKeyword ||
200
- (ts.isIdentifier(node.expression) && node.expression.text === "require"))) {
201
- const value = literal(node.arguments[0]);
202
- if (value === null)
463
+ if (ts.isCallExpression(node)) {
464
+ const module = loaded(node);
465
+ if (module !== undefined) {
466
+ const value = literal(node.arguments[0]);
467
+ if (value === null)
468
+ unresolved.push(at(node));
469
+ else
470
+ members.push(...spellings(value));
471
+ }
472
+ break;
473
+ }
474
+ // A loader handed on (const r = [require][0]) loads under a name Kibi does not follow.
475
+ if (ts.isIdentifier(node) || ts.isPropertyAccessExpression(node)) {
476
+ const name = canonical(chain(node) ?? "");
477
+ if ((LOADERS.has(name) || REQUIRE_FACTORIES.has(name)) &&
478
+ maximal(node) &&
479
+ !calledAs(node) &&
480
+ !declares(node) &&
481
+ !inType(node) &&
482
+ !followed(node))
203
483
  unresolved.push(at(node));
204
- else
205
- members.push(value);
206
484
  }
207
485
  break;
208
486
  }
209
487
  case "calls": {
210
488
  if (ts.isCallExpression(node)) {
211
489
  const target = node.expression;
212
- const name = chain(target);
213
- if (name !== null)
214
- members.push(name);
490
+ const written = chain(target);
491
+ if (written !== null) {
492
+ const listed = [written, canonical(written)];
493
+ // f.call(...), f.apply(...) and f.bind(...)() call f.
494
+ for (const name of [...listed]) {
495
+ const bound = /^(.*)\.(?:call|apply|bind)$/.exec(name);
496
+ if (bound?.[1])
497
+ listed.push(bound[1]);
498
+ }
499
+ members.push(...new Set(listed));
500
+ }
215
501
  else if (ts.isElementAccessExpression(target))
216
502
  unresolved.push(at(target));
217
503
  }
504
+ else if ((ts.isIdentifier(node) || ts.isPropertyAccessExpression(node) || ts.isElementAccessExpression(node)) &&
505
+ maximal(node) &&
506
+ !calledAs(node) &&
507
+ !declares(node) &&
508
+ !inType(node)) {
509
+ const written = chain(node);
510
+ const head = written?.split(".")[0];
511
+ if (written !== null && head !== undefined && names.has(head) && !followed(node)) {
512
+ const name = canonical(written);
513
+ if (moduleNames.has(head) && written === head && NODE_BUILTINS.has(name)) {
514
+ // A built-in module handed on: any of its members may be called.
515
+ members.push(`${name}.*`);
516
+ }
517
+ else if (name.includes(".") || written !== name) {
518
+ // An imported function used as a value is listed as called.
519
+ members.push(...new Set([written, name]));
520
+ }
521
+ }
522
+ else if (written !== null && head !== undefined && names.has(head) && followed(node)) {
523
+ const name = canonical(written);
524
+ if (name !== written && name.includes("."))
525
+ members.push(name);
526
+ }
527
+ }
218
528
  break;
219
529
  }
220
530
  case "string_args": {
221
531
  if (ts.isCallExpression(node)) {
222
- const name = chain(node.expression);
223
- if (name !== null && (name === callee || (!callee.includes(".") && name.endsWith(`.${callee}`)))) {
532
+ const written = chain(node.expression);
533
+ const listed = written === null ? [] : [...new Set([written, canonical(written)])];
534
+ if (listed.some((name) => name === callee || (!callee.includes(".") && name.endsWith(`.${callee}`)))) {
224
535
  const value = literal(node.arguments[0]);
225
536
  if (value === null)
226
537
  unresolved.push(at(node));
@@ -261,6 +572,31 @@ export function extractScriptReferences(ts, file, text, spec) {
261
572
  }
262
573
  ts.forEachChild(node, visit);
263
574
  };
575
+ // Destructuring from the environment, process or the global object:
576
+ // `const { PORT } = process.env` reads PORT, `const { env: { HOST } } =
577
+ // process` reads HOST; a rest element or a computed key could read any.
578
+ const readPattern = (pattern, from) => {
579
+ if (ts.isIdentifier(pattern))
580
+ return;
581
+ if (!ts.isObjectBindingPattern(pattern)) {
582
+ unresolved.push(at(pattern));
583
+ return;
584
+ }
585
+ for (const element of pattern.elements) {
586
+ const property = element.propertyName ?? element.name;
587
+ if (element.dotDotDotToken || !(ts.isIdentifier(property) || ts.isStringLiteral(property))) {
588
+ unresolved.push(at(element));
589
+ continue;
590
+ }
591
+ const path = canonical(`${from}.${property.text}`);
592
+ if (ENV_OBJECTS.has(from))
593
+ members.push(property.text);
594
+ else if (ENV_OBJECTS.has(path) || path === "process" || GLOBAL_OBJECTS.has(path)) {
595
+ if (!ts.isIdentifier(element.name))
596
+ readPattern(element.name, path);
597
+ }
598
+ }
599
+ };
264
600
  visit(source);
265
601
  return { members, unresolved };
266
602
  }
@@ -278,8 +614,110 @@ def chain(node):
278
614
  return None
279
615
  def literal(node):
280
616
  return node.value if isinstance(node, ast.Constant) and isinstance(node.value, str) else None
281
- ENV_GETTERS = {"os.getenv", "getenv", "os.environ.get", "environ.get"}
282
- ENV_OBJECTS = {"os.environ", "environ"}
617
+ # Bare getenv and environ are read as the environment even without an import.
618
+ ENV_GETTERS = {"os.getenv", "os.getenvb", "getenv"}
619
+ ENV_OBJECTS = {"os.environ", "os.environb", "environ", "posix.environ", "nt.environ"}
620
+ ENV_KEY_METHODS = {"get", "setdefault", "pop"}
621
+ ENV_TEMPLATES = {"os.path.expandvars", "posixpath.expandvars", "ntpath.expandvars"}
622
+ # Code run from text, or a module looked up at run time, can read or call anything.
623
+ IMPORTERS = {"__import__", "importlib.import_module", "import_module", "builtins.__import__"}
624
+ DYNAMIC = IMPORTERS | {"exec", "eval", "compile", "builtins.exec", "builtins.eval", "builtins.compile",
625
+ "pkgutil.resolve_name", "pkgutil.get_loader", "pkgutil.find_loader", "importlib.reload",
626
+ "importlib.__import__", "importlib.find_loader",
627
+ # Unpickling calls whatever the data names (os.system).
628
+ "pickle.loads", "pickle.load", "pickle.Unpickler", "_pickle.loads", "_pickle.load",
629
+ "_pickle.Unpickler", "marshal.loads", "marshal.load", "shelve.open", "dill.loads",
630
+ "dill.load", "cloudpickle.loads", "cloudpickle.load", "jsonpickle.decode", "yaml.unsafe_load",
631
+ "yaml.unsafe_load_all"}
632
+ DYNAMIC_PREFIXES = ("runpy.", "importlib.util.", "importlib.machinery.", "importlib._bootstrap", "zipimport.")
633
+ def dynamic(name):
634
+ return name is not None and (name in DYNAMIC or name.startswith(DYNAMIC_PREFIXES))
635
+ # A module's namespace read as a dictionary: any module, or any of its members, under any key.
636
+ NAMESPACES = {"globals", "locals", "builtins.globals", "builtins.locals"}
637
+ # Modules that read the environment, run programs or load code. One reached
638
+ # as an attribute of another module (shutil.os, path.os, from .main import os)
639
+ # is that module under a name Kibi does not follow, so it fails the read.
640
+ REACHED = {"os", "posix", "nt", "sys", "subprocess", "builtins", "importlib", "runpy", "pkgutil", "shutil", "pty", "socket", "ctypes"}
641
+ ENV_ATTRS = {"environ", "environb", "getenv", "getenvb"}
642
+ ENV_MODULES = {"os", "posix", "nt"}
643
+ # The modules a claim names: their own private members (os._exit) are read as written.
644
+ NAMED_MODULES = {"os", "posix", "nt", "sys", "subprocess"}
645
+ # Foreign function interfaces: a library they load (libc's getenv or system)
646
+ # runs native code Kibi cannot read.
647
+ FFI_MODULES = {"ctypes", "_ctypes", "cffi", "_cffi_backend"}
648
+ def private(name):
649
+ return name.startswith("_") and not (name.startswith("__") and name.endswith("__"))
650
+ def hidden(parts, i):
651
+ # A private name of a module: tempfile._os and threading._os are os, and
652
+ # what another standard module keeps under a private name is not known here.
653
+ part = parts[i]
654
+ if not private(part):
655
+ return False
656
+ return part.lstrip("_") in REACHED or (parts[0] in STDLIB and ".".join(parts[:i]) not in NAMED_MODULES)
657
+ NAMESPACE_ATTRS = {"__dict__", "__getattribute__", "__getattr__", "__loader__", "__spec__", "__builtins__", "__import__", "__globals__"}
658
+ STDLIB = getattr(sys, "stdlib_module_names", frozenset())
659
+ def _system_names():
660
+ import os as _os, subprocess as _subprocess
661
+ found = set()
662
+ for module in (_os, _subprocess, sys):
663
+ for name in dir(module):
664
+ value = getattr(module, name, None)
665
+ if not name.startswith("_") and callable(value) and not (isinstance(value, type) and issubclass(value, BaseException)):
666
+ found.add(name)
667
+ return found | ENV_ATTRS
668
+ # What a name a project module re-exports may stand for when it is os,
669
+ # subprocess or sys, or one of their functions.
670
+ SYSTEM_NAMES = _system_names()
671
+ # os re-exports posix (nt on Windows): posix.system is os.system, so a name
672
+ # through posix or nt is listed as the os name too.
673
+ def as_os(name):
674
+ head, dot, rest = name.partition(".")
675
+ return "os" + dot + rest if head in ("posix", "nt") else None
676
+ def with_os(names):
677
+ return list(dict.fromkeys(list(names) + [n for n in map(as_os, names) if n is not None]))
678
+ # Whether a standard-library dotted name is a module object (os.path,
679
+ # importlib.resources, shutil.fnmatch), found by importing the standard
680
+ # module it starts with. None when that is not known: a name outside the
681
+ # standard library, or one whose module does not import here.
682
+ MODULE_CACHE = {}
683
+ # Standard modules that act when imported (print, open a browser): not imported here.
684
+ NOISY = {"this", "antigravity", "__hello__", "__phello__", "idlelib", "turtledemo"}
685
+ def is_module(dotted):
686
+ parts = dotted.split(".")
687
+ if not parts[0] or parts[0] not in STDLIB:
688
+ return None
689
+ if len(parts) == 1:
690
+ return True
691
+ if parts[0] in NOISY:
692
+ return None
693
+ if dotted not in MODULE_CACHE:
694
+ import contextlib, importlib, io, types
695
+ try:
696
+ with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
697
+ value = importlib.import_module(parts[0])
698
+ for i in range(1, len(parts)):
699
+ if hasattr(value, parts[i]):
700
+ value = getattr(value, parts[i])
701
+ elif isinstance(value, types.ModuleType) and hasattr(value, "__path__"):
702
+ try:
703
+ value = importlib.import_module(".".join(parts[:i + 1]))
704
+ except ModuleNotFoundError:
705
+ value = None
706
+ break
707
+ else:
708
+ value = None
709
+ break
710
+ MODULE_CACHE[dotted] = isinstance(value, types.ModuleType)
711
+ except Exception:
712
+ # A module that does not import here (winreg on Linux) is not known,
713
+ # as a project module is not.
714
+ MODULE_CACHE[dotted] = None
715
+ return MODULE_CACHE[dotted]
716
+ def prefixes(dots, dotted):
717
+ parts = dotted.split(".") if dotted else []
718
+ return [dots + ".".join(parts[:i]) for i in range(1, len(parts) + 1)] or [dots]
719
+ def joined(base, name):
720
+ return base + name if base.endswith(".") else base + "." + name
283
721
  for file in plan["files"]:
284
722
  with open(file, encoding="utf-8") as handle:
285
723
  text = handle.read()
@@ -293,36 +731,219 @@ for file in plan["files"]:
293
731
  line = getattr(node, "lineno", 1)
294
732
  unresolved.append({"file": file, "line": line, "text": (lines[line - 1].strip() if line - 1 < len(lines) else "")[:80]})
295
733
  caught = set()
734
+ parent = {}
735
+ # What each name the file imports stands for: import subprocess as sp makes
736
+ # sp stand for subprocess, from os import getenv as g makes g os.getenv.
737
+ names, modules = {}, set()
738
+ # Names bound by an import from a module outside the standard library
739
+ # (a project or third-party module): what they are is not known here.
740
+ foreign, imported_as = set(), {}
296
741
  for node in ast.walk(tree):
742
+ for child in ast.iter_child_nodes(node):
743
+ parent[child] = node
297
744
  if isinstance(node, ast.ExceptHandler) and node.name:
298
745
  caught.add(node.name)
746
+ elif isinstance(node, ast.Import):
747
+ for alias in node.names:
748
+ local = alias.asname or alias.name.split(".")[0]
749
+ names[local] = alias.name if alias.asname else local
750
+ modules.add(local)
751
+ if alias.name.split(".")[0] not in STDLIB:
752
+ foreign.add(local)
753
+ elif isinstance(node, ast.ImportFrom):
754
+ base = "." * node.level + (node.module or "")
755
+ outside = node.level > 0 or (node.module or "").split(".")[0] not in STDLIB
756
+ for alias in node.names:
757
+ if alias.name != "*":
758
+ local = alias.asname or alias.name
759
+ names[local] = joined(base, alias.name)
760
+ imported_as[local] = alias.name
761
+ if outside:
762
+ foreign.add(local)
763
+ def canonical(name):
764
+ head, dot, rest = name.partition(".")
765
+ return names[head] + dot + rest if head in names else name
766
+ def spelled(node):
767
+ name = chain(node)
768
+ return None if name is None else (name, canonical(name))
769
+ def called(node):
770
+ up = parent.get(node)
771
+ return isinstance(up, ast.Call) and up.func is node
772
+ def maximal(node):
773
+ up = parent.get(node)
774
+ return not (isinstance(up, ast.Attribute) and up.value is node)
775
+ def reaches(node):
776
+ # A chain through an imported name that reaches a module Kibi does not
777
+ # follow under that spelling, or a module's namespace.
778
+ written = chain(node)
779
+ if written is None:
780
+ return False
781
+ head = written.split(".")[0]
782
+ parts = canonical(written).split(".")
783
+ if head == "__builtins__" or "__globals__" in parts or "__builtins__" in parts:
784
+ return True
785
+ if head not in names:
786
+ return False
787
+ for i in range(1, len(parts)):
788
+ part = parts[i]
789
+ if hidden(parts, i):
790
+ return True
791
+ if part in REACHED and not (part == "subprocess" and parts[:i] == ["asyncio"]):
792
+ return True
793
+ if part in NAMESPACE_ATTRS:
794
+ return True
795
+ if extract == "env" and part in ENV_ATTRS and ".".join(parts[:i]) not in ENV_MODULES:
796
+ return True
797
+ if extract == "calls" and head in foreign:
798
+ # A project module's name may be os, subprocess or sys, or one of
799
+ # their functions, re-exported: from .main import os as o; o.system().
800
+ after = written.split(".")[1:]
801
+ if head in imported_as:
802
+ after = [imported_as[head]] + after
803
+ return any(part in SYSTEM_NAMES for part in after)
804
+ return False
805
+ def flows(node):
806
+ # An imported module used as a value (assigned, passed, returned or put
807
+ # in a container), not as the head of a chain or a call. A name from a
808
+ # project or third-party module may be a module too, so assigning one
809
+ # to another name fails the read as well.
810
+ if not isinstance(node.ctx, ast.Load) or not maximal(node) or called(node):
811
+ return False
812
+ written = chain(node)
813
+ if written is None or written.split(".")[0] not in names:
814
+ return False
815
+ if isinstance(node, ast.Name) and node.id in modules:
816
+ return True
817
+ known = is_module(canonical(written))
818
+ if known is not None:
819
+ return known
820
+ up = parent.get(node)
821
+ return isinstance(up, (ast.Assign, ast.AnnAssign, ast.NamedExpr, ast.AugAssign)) and up.value is node
822
+ def ffi(node):
823
+ written = chain(node)
824
+ return written is not None and written.split(".")[0] in names and canonical(written).split(".")[0] in FFI_MODULES
825
+ def hides(node):
826
+ # from tempfile import _os: a module under a private name of another one.
827
+ if not isinstance(node, ast.ImportFrom):
828
+ return False
829
+ base = "." * node.level + (node.module or "")
830
+ parts = base.split(".") if node.level == 0 else [base]
831
+ return any(alias.name != "*" and hidden(parts + [alias.name], len(parts)) for alias in node.names)
832
+ def env_use(node, name):
833
+ up = parent.get(node)
834
+ if name in ENV_GETTERS:
835
+ if called(node):
836
+ value = literal(up.args[0]) if up.args else None
837
+ return members.append(value) if value is not None else miss(node)
838
+ return miss(node)
839
+ if isinstance(up, ast.Subscript) and up.value is node:
840
+ value = literal(up.slice)
841
+ return members.append(value) if value is not None else miss(node)
842
+ if isinstance(up, ast.Attribute) and up.attr in ENV_KEY_METHODS and called(up):
843
+ call = parent[up]
844
+ value = literal(call.args[0]) if call.args else None
845
+ return members.append(value) if value is not None else miss(node)
846
+ if (isinstance(up, ast.Compare) and len(up.ops) == 1 and isinstance(up.ops[0], (ast.In, ast.NotIn))
847
+ and up.comparators[0] is node and literal(up.left) is not None):
848
+ return members.append(literal(up.left))
849
+ # The whole environment handed on, copied, iterated or assigned.
850
+ miss(node)
299
851
  for node in ast.walk(tree):
852
+ if extract in ("env", "calls", "imports") and isinstance(node, ast.Call):
853
+ target = spelled(node.func)
854
+ name = None if target is None else target[1]
855
+ if dynamic(name) and not (extract == "imports" and name in IMPORTERS):
856
+ miss(node)
857
+ continue
858
+ if name in NAMESPACES or (name in ("vars", "builtins.vars") and (
859
+ not node.args or (chain(node.args[0]) or "").split(".")[0] in names)):
860
+ # globals()['os'], vars(), vars(module): a namespace read by key,
861
+ # which holds modules (vars(tempfile)["_os"] is os).
862
+ miss(node)
863
+ continue
864
+ if name in ("getattr", "builtins.getattr") and node.args:
865
+ inner = chain(node.args[0])
866
+ if inner is not None and inner.split(".")[0] in names:
867
+ # A member of an imported module looked up by name.
868
+ miss(node)
869
+ continue
870
+ if extract in ("env", "calls", "imports") and isinstance(node, ast.Attribute) and canonical(chain(node) or "") == "sys.modules":
871
+ miss(node)
872
+ continue
873
+ if extract in ("env", "calls", "imports") and hides(node):
874
+ miss(node)
875
+ continue
876
+ if (extract in ("env", "calls") and isinstance(node, ast.ImportFrom) and node.level == 0
877
+ and (node.module or "").split(".")[0] in FFI_MODULES and any(alias.name == "*" for alias in node.names)):
878
+ miss(node)
879
+ continue
880
+ if extract in ("env", "calls") and isinstance(node, (ast.Name, ast.Attribute)) and maximal(node) and ffi(node):
881
+ # ctypes.CDLL(None).getenv(b"X"): a foreign call can read or run anything.
882
+ miss(node)
883
+ continue
884
+ if extract in ("env", "calls", "imports") and isinstance(node, (ast.Name, ast.Attribute)) and maximal(node) and reaches(node):
885
+ miss(node)
886
+ continue
887
+ if extract in ("env", "calls", "imports") and isinstance(node, (ast.Name, ast.Attribute)) and flows(node):
888
+ # _t = tempfile; _t._os: a module handed on is used under a name Kibi does not follow.
889
+ miss(node)
890
+ continue
300
891
  if extract == "env":
301
- if isinstance(node, ast.Call) and chain(node.func) in ENV_GETTERS:
302
- value = literal(node.args[0]) if node.args else None
303
- (members.append(value) if value is not None else miss(node))
304
- elif isinstance(node, ast.Subscript) and chain(node.value) in ENV_OBJECTS:
305
- value = literal(node.slice)
306
- (members.append(value) if value is not None else miss(node))
892
+ if isinstance(node, ast.ImportFrom) and node.module == "os" and any(alias.name == "*" for alias in node.names):
893
+ miss(node)
894
+ elif isinstance(node, (ast.Name, ast.Attribute)):
895
+ target = spelled(node)
896
+ if target is None:
897
+ continue
898
+ name = target[1]
899
+ if name in ENV_GETTERS or name in ENV_OBJECTS:
900
+ env_use(node, name)
901
+ elif name in ENV_TEMPLATES:
902
+ miss(node)
903
+ elif isinstance(node, ast.Name) and (name in ENV_MODULES or node.id in modules) and maximal(node):
904
+ # An imported module itself handed on: its later reads could use any name.
905
+ miss(node)
307
906
  elif extract == "imports":
308
907
  if isinstance(node, ast.Import):
309
- members.extend(alias.name for alias in node.names)
908
+ for alias in node.names:
909
+ members.extend(with_os(prefixes("", alias.name)))
310
910
  elif isinstance(node, ast.ImportFrom):
311
- members.append("." * node.level + (node.module or ""))
312
- elif isinstance(node, ast.Call) and chain(node.func) in {"importlib.import_module", "import_module", "__import__"}:
911
+ if any(alias.name in REACHED for alias in node.names) and not (node.level == 0 and node.module == "asyncio"):
912
+ # from .main import os: a module reached through another one,
913
+ # under a spelling a claim about the module would not match.
914
+ miss(node)
915
+ dots = "." * node.level
916
+ members.extend(with_os(prefixes(dots, node.module or "")))
917
+ base = dots + (node.module or "")
918
+ members.extend(with_os([joined(base, alias.name) for alias in node.names if alias.name != "*"]))
919
+ elif isinstance(node, ast.Call) and canonical(chain(node.func) or "") in IMPORTERS:
313
920
  value = literal(node.args[0]) if node.args else None
314
921
  (members.append(value) if value is not None else miss(node))
315
922
  elif extract == "calls":
316
- if isinstance(node, ast.Call):
317
- name = chain(node.func)
318
- if name is not None:
319
- members.append(name)
923
+ if isinstance(node, ast.ImportFrom) and any(alias.name == "*" for alias in node.names):
924
+ # A star import could bring any name into scope under a bare spelling.
925
+ miss(node)
926
+ elif isinstance(node, ast.Call):
927
+ target = spelled(node.func)
928
+ if target is not None:
929
+ # A call through an alias is listed under what it names, too.
930
+ members.extend(with_os(target))
320
931
  elif isinstance(node.func, ast.Subscript):
321
932
  miss(node)
933
+ elif isinstance(node, (ast.Name, ast.Attribute)) and isinstance(node.ctx, ast.Load) and maximal(node) and not called(node):
934
+ target = spelled(node)
935
+ if target is not None and target[0].split(".")[0] in names:
936
+ if isinstance(node, ast.Name) and node.id in modules:
937
+ # An imported module handed on: its later calls could use any name.
938
+ miss(node)
939
+ else:
940
+ # An imported function used as a value is listed as called.
941
+ members.extend(with_os(target))
322
942
  elif extract == "string_args":
323
943
  if isinstance(node, ast.Call):
324
- name = chain(node.func)
325
- if name is not None and (name == callee or ("." not in callee and name.endswith("." + callee))):
944
+ target = spelled(node.func)
945
+ target = None if target is None else with_os(target)
946
+ if target is not None and any(name == callee or ("." not in callee and name.endswith("." + callee)) for name in target):
326
947
  value = literal(node.args[0]) if node.args else None
327
948
  (members.append(value) if value is not None else miss(node))
328
949
  elif extract == "thrown":
@@ -344,7 +965,8 @@ for file in plan["files"]:
344
965
  if keyword.arg == key:
345
966
  value = literal(keyword.value)
346
967
  (members.append(value) if value is not None else miss(keyword.value))
347
- print(json.dumps({"ok": True, "members": members, "unresolved": unresolved}))
968
+ unique = list({(row["file"], row["line"], row["text"]): row for row in unresolved}.values())
969
+ print(json.dumps({"ok": True, "members": members, "unresolved": unique}))
348
970
  `;
349
971
  const EXTRACT_WHAT = {
350
972
  env: "environment variable reads",
@@ -358,15 +980,26 @@ async function readReferences(workspaceRoot, spec) {
358
980
  const files = matchWorkspaceFiles(workspaceRoot, globList(spec.files), globList(spec.ignore));
359
981
  if (files.length === 0)
360
982
  return { ok: false, reason: `no file matches ${globList(spec.files).join(", ")}` };
361
- const other = files.filter((file) => !JS_FILE.test(file) && !PY_FILE.test(file));
362
- if (other.length > 0)
983
+ const other = files.filter((file) => {
984
+ const language = codeLanguage(file);
985
+ return language !== "js" && language !== "python" && language !== "prolog";
986
+ });
987
+ const unknown = other.filter((file) => pluginReferenceLanguage(file) === null);
988
+ if (unknown.length > 0)
363
989
  return {
364
990
  ok: false,
365
- reason: `the references reader reads TypeScript, JavaScript and Python files; narrow files to exclude ${other.slice(0, 3).join(", ")}`,
991
+ reason: `${unsupportedLanguageReason("the references reader", ["js", "python", "prolog"], unknown).replace(/ files, not /, ` files (and Go, Java, C# and Rust through ${REFERENCE_PLUGIN_PACKAGE}), not `)}; Kibi cannot decide a references claim for those files, so the claim stays unproved (do not drop them from files: a claim about them would then be checked on other code)`,
366
992
  };
367
993
  const members = [];
368
994
  const unresolved = [];
369
- const scripts = files.filter((file) => JS_FILE.test(file));
995
+ if (other.length > 0) {
996
+ const read = await readPluginReferences(workspaceRoot, other, spec);
997
+ if (!read.ok)
998
+ return read;
999
+ members.push(...read.members);
1000
+ unresolved.push(...read.unresolved);
1001
+ }
1002
+ const scripts = files.filter((file) => codeLanguage(file) === "js");
370
1003
  if (scripts.length > 0) {
371
1004
  const ts = loadTypeScript(workspaceRoot);
372
1005
  if (ts === null)
@@ -380,9 +1013,9 @@ async function readReferences(workspaceRoot, spec) {
380
1013
  unresolved.push(...found.unresolved);
381
1014
  }
382
1015
  }
383
- const python = files.filter((file) => PY_FILE.test(file));
1016
+ const python = files.filter((file) => codeLanguage(file) === "python");
384
1017
  if (python.length > 0) {
385
- const run = runCodeEvaluator(workspaceRoot, python[0], { js: "", python: PYTHON_REFERENCES }, [JSON.stringify({ extract: spec.extract, callee: spec.callee, key: spec.key, files: python })], (parsed) => parsed !== null && typeof parsed === "object" && "ok" in parsed);
1018
+ const run = runCodeEvaluator(workspaceRoot, python[0], { python: PYTHON_REFERENCES }, [JSON.stringify({ extract: spec.extract, callee: spec.callee, key: spec.key, files: python })], (parsed) => parsed !== null && typeof parsed === "object" && "ok" in parsed);
386
1019
  if (!run.ok)
387
1020
  return { ok: false, reason: `Kibi could not run Python to read ${python[0]}${run.failure ? `: ${run.failure}` : ""}` };
388
1021
  const value = run.value;
@@ -391,10 +1024,179 @@ async function readReferences(workspaceRoot, spec) {
391
1024
  members.push(...(value.members ?? []));
392
1025
  unresolved.push(...(value.unresolved ?? []));
393
1026
  }
1027
+ const prolog = files.filter((file) => codeLanguage(file) === "prolog");
1028
+ if (prolog.length > 0) {
1029
+ if (spec.extract === "property")
1030
+ return {
1031
+ ok: false,
1032
+ reason: `SWI-Prolog has no property values to extract (${prolog[0]}); read fact arguments with the prolog reader`,
1033
+ };
1034
+ const read = runPrologReader(workspaceRoot, prolog[0], {
1035
+ mode: "references",
1036
+ extract: spec.extract,
1037
+ ...(typeof spec.callee === "string" ? { callee: spec.callee } : {}),
1038
+ files: prolog,
1039
+ });
1040
+ if (!read.ok)
1041
+ return read;
1042
+ members.push(...(read.value.members ?? []));
1043
+ unresolved.push(...(read.value.unresolved ?? []));
1044
+ }
394
1045
  if (unresolved.length > 0)
395
1046
  return { ok: false, reason: unresolvedReason(EXTRACT_WHAT[String(spec.extract)] ?? "references", unresolved) };
396
1047
  return { ok: true, members };
397
1048
  }
1049
+ const PLUGIN_READ_TIMEOUT_MS = 10_000;
1050
+ /**
1051
+ * Read references of Go, Java, C# and Rust files through a qualified
1052
+ * kibi.reference-extractor.v1 provider. Without one the read is refused with
1053
+ * what to install, never answered as empty.
1054
+ */
1055
+ // implements REQ-kibi-tree-sitter-references
1056
+ async function readPluginReferences(workspaceRoot, files, spec) {
1057
+ const { extractors, problems } = await proofReferenceExtractorLoader.load(workspaceRoot);
1058
+ // Callers pass only files a reference reader covers (unknown ones are refused first).
1059
+ const languages = [...new Set(files.flatMap((file) => pluginReferenceLanguage(file) ?? []))];
1060
+ const unread = files.filter((file) => !extractors.some((extractor) => extractor.supports({ path: file })));
1061
+ if (unread.length > 0) {
1062
+ const shown = `${unread.slice(0, 3).join(", ")}${unread.length > 3 ? ` and ${unread.length - 3} more` : ""}`;
1063
+ const why = problems.length > 0 ? ` (${problems.join("; ")})` : "";
1064
+ return {
1065
+ ok: false,
1066
+ reason: `the references reader reads ${languages.join(", ")} files through ${REFERENCE_PLUGIN_PACKAGE}, which is not active for proofs here${why}; add it to devDependencies and activate kibi.reference-extractor.v1 in package.json#kibi.plugins (kibi doctor lists what this project needs): ${shown}`,
1067
+ };
1068
+ }
1069
+ const members = [];
1070
+ const unresolved = [];
1071
+ const shadowRows = createShadowRowFilter(workspaceRoot);
1072
+ for (const file of files) {
1073
+ const extractor = extractors.find((candidate) => candidate.supports({ path: file }));
1074
+ if (extractor === undefined)
1075
+ continue;
1076
+ let result;
1077
+ try {
1078
+ result = validateReferenceExtractionResult(await extractor.extract({
1079
+ path: file,
1080
+ content: readFileSync(path.join(workspaceRoot, file), "utf8"),
1081
+ extract: spec.extract,
1082
+ ...(typeof spec.callee === "string" ? { callee: spec.callee } : {}),
1083
+ ...(typeof spec.key === "string" ? { key: spec.key } : {}),
1084
+ timeoutMs: PLUGIN_READ_TIMEOUT_MS,
1085
+ }));
1086
+ }
1087
+ catch (error) {
1088
+ return {
1089
+ ok: false,
1090
+ reason: `Kibi could not read ${file}: ${error instanceof Error ? error.message : String(error)}`,
1091
+ };
1092
+ }
1093
+ if (result.status !== "ok")
1094
+ return { ok: false, reason: `Kibi could not read ${file}: ${result.message}` };
1095
+ members.push(...result.members);
1096
+ // A row about a name another file may bind stays only if one does.
1097
+ for (const row of shadowRows(file, result.unresolved))
1098
+ unresolved.push({ file, line: row.line, text: row.text });
1099
+ }
1100
+ // A C# file also imports what the project's global and implicit usings
1101
+ // bring in, which no single file shows.
1102
+ if (spec.extract === "imports" && files.some((file) => /\.cs$/i.test(file)))
1103
+ members.push(...csharpProjectUsings(workspaceRoot));
1104
+ return { ok: true, members, unresolved };
1105
+ }
1106
+ /** The namespaces `<ImplicitUsings>enable</ImplicitUsings>` adds, by SDK. */
1107
+ const CSHARP_IMPLICIT_USINGS = [
1108
+ "System",
1109
+ "System.Collections.Generic",
1110
+ "System.IO",
1111
+ "System.Linq",
1112
+ "System.Net.Http",
1113
+ "System.Threading",
1114
+ "System.Threading.Tasks",
1115
+ ];
1116
+ const CSHARP_WEB_IMPLICIT_USINGS = [
1117
+ "System.Net.Http.Json",
1118
+ "Microsoft.AspNetCore.Builder",
1119
+ "Microsoft.AspNetCore.Hosting",
1120
+ "Microsoft.AspNetCore.Http",
1121
+ "Microsoft.AspNetCore.Routing",
1122
+ "Microsoft.Extensions.Configuration",
1123
+ "Microsoft.Extensions.DependencyInjection",
1124
+ "Microsoft.Extensions.Hosting",
1125
+ "Microsoft.Extensions.Logging",
1126
+ ];
1127
+ /**
1128
+ * Namespaces every C# file of the project imports: `global using`
1129
+ * directives in any tracked .cs file, `<Using Include>` items and the SDK's
1130
+ * implicit usings in any tracked .csproj.
1131
+ */
1132
+ // implements REQ-kibi-tree-sitter-references
1133
+ export function csharpProjectUsings(workspaceRoot) {
1134
+ let tracked;
1135
+ try {
1136
+ tracked = execFileSync("git", ["ls-files", "-z", "--", "*.cs", "*.csproj", "*.props"], {
1137
+ cwd: workspaceRoot,
1138
+ encoding: "utf8",
1139
+ stdio: ["ignore", "pipe", "ignore"],
1140
+ maxBuffer: 64 * 1024 * 1024,
1141
+ })
1142
+ .split("\0")
1143
+ .filter((file) => file.length > 0);
1144
+ }
1145
+ catch {
1146
+ tracked = [];
1147
+ }
1148
+ const usings = new Set();
1149
+ for (const file of tracked) {
1150
+ let text;
1151
+ try {
1152
+ text = readFileSync(path.join(workspaceRoot, file), "utf8");
1153
+ }
1154
+ catch {
1155
+ continue;
1156
+ }
1157
+ if (/\.cs$/i.test(file)) {
1158
+ for (const match of text.matchAll(/^\s*global\s+using\s+(?:static\s+)?(?:(\w+)\s*=\s*)?([\w.:]+)\s*;/gm))
1159
+ usings.add((match[2] ?? "").replace(/^global::/, ""));
1160
+ continue;
1161
+ }
1162
+ for (const match of text.matchAll(/<Using\s+Include="([^"]+)"/g))
1163
+ usings.add(match[1] ?? "");
1164
+ if (/<ImplicitUsings>\s*(?:enable|true)\s*<\/ImplicitUsings>/i.test(text)) {
1165
+ for (const name of CSHARP_IMPLICIT_USINGS)
1166
+ usings.add(name);
1167
+ if (/Sdk="Microsoft\.NET\.Sdk\.Web"/.test(text))
1168
+ for (const name of CSHARP_WEB_IMPLICIT_USINGS)
1169
+ usings.add(name);
1170
+ }
1171
+ }
1172
+ usings.delete("");
1173
+ return [...usings].sort();
1174
+ }
1175
+ /** Run the SWI-Prolog source reader, which reads terms without running them. */
1176
+ // implements REQ-kibi-prolog-source-readers
1177
+ function runPrologReader(workspaceRoot, file, plan) {
1178
+ const run = runCodeEvaluator(workspaceRoot, file, { prolog: PROLOG_SOURCE_READER }, [JSON.stringify(plan)], (parsed) => parsed !== null && typeof parsed === "object" && "ok" in parsed);
1179
+ if (!run.ok)
1180
+ return { ok: false, reason: `Kibi could not run SWI-Prolog to read ${file}${run.failure ? `: ${run.failure}` : ""}` };
1181
+ const value = run.value;
1182
+ if (!value.ok)
1183
+ return { ok: false, reason: value.error ?? `Kibi could not read ${file}` };
1184
+ return { ok: true, value };
1185
+ }
1186
+ /** The prolog reader: one argument of every fact with a functor, in file order. */
1187
+ // implements REQ-kibi-prolog-source-readers
1188
+ function readPrologTerms(workspaceRoot, spec) {
1189
+ const file = String(spec.file);
1190
+ const read = runPrologReader(workspaceRoot, file, {
1191
+ mode: "terms",
1192
+ file,
1193
+ functor: spec.functor,
1194
+ ...(spec.arg !== undefined ? { arg: spec.arg } : {}),
1195
+ });
1196
+ if (!read.ok)
1197
+ return read;
1198
+ return { ok: true, members: read.value.members ?? [], sequence: read.value.sequence ?? [] };
1199
+ }
398
1200
  // Built on first use so TS_RESOLVE_HOOKS is initialized.
399
1201
  const jsModuleEvaluator = () => `
400
1202
  const plan = JSON.parse(process.argv.at(-1));
@@ -693,6 +1495,8 @@ function evaluateModule(workspaceRoot, spec) {
693
1495
  export const evaluateSourceCode = async (workspaceRoot, spec) => {
694
1496
  if (spec.reader === "references")
695
1497
  return readReferences(workspaceRoot, spec);
1498
+ if (spec.reader === "prolog")
1499
+ return readPrologTerms(workspaceRoot, spec);
696
1500
  if (spec.reader === "schema" && spec.format !== "drizzle")
697
1501
  return { ok: false, reason: `format ${String(spec.format)} is read without code` };
698
1502
  return evaluateModule(workspaceRoot, spec);