@descryy/adapter-python 0.4.1 → 0.5.0

package/dist/extract.js

@@ -30,7 +30,7 @@
  * because `a/__init__.py` importing from `b/__init__.py` importing back is legal
  * Python and must not hang the run.
  */
-import { databaseTableNodesFromModel, edgeId, endpointQsp, isCallableNodeType, isTypeLikeNodeType, nodeId, normaliseEndpointPath, symbolQsp, testCaseQsp, } from "@descryy/ir";
+import { databaseTableNodesFromModel, edgeId, endpointQsp, externalQsp, isCallableNodeType, isTypeLikeNodeType, nodeId, normaliseEndpointPath, symbolQsp, testCaseQsp, } from "@descryy/ir";
 import { importRoots, modulePathOf, moduleNameOf, resolveImport, } from "./module.js";
 import { attributeExternal, externalImportTable, ownNamespaces, STDLIB_BASIS, } from "./external.js";
 import { DECLARATIVE_BASE_FACTORIES, ORM_BASE_MODULES, isFrameworkBaseName, ormShapeOf, } from "./orm.js";
@@ -39,7 +39,16 @@ import { drfShapeOf, dtoShapeOf, isDrfSerializerRoot, isPydanticRoot, isValidato
 import { DRF_ACTIONS, drfRegisterCallsIn, drfRouterLocalsIn, djangoUrlEntriesIn, HTTP_METHODS, isApplicationRouter, joinPath, mountsIn, odooRouteFromDecorator, routeFromDecorator, routersIn, } from "./routes.js";
 import { GRAPHQL_METHOD, fieldFromMethod, fieldsFromAttributes, operationPath, schemaDeclsIn, } from "./graphql.js";
 import { clientCallsIn, clientLocalsIn } from "./client.js";
+import { modelParsing, validateCalls } from "./response-shape.js";
 export const LANGUAGE = "python";
+/** Which packaging world an external module name belongs to. The name is the
+ *  ecosystem's, not an installed path's: `pyyaml` installs `yaml`, and `yaml`
+ *  is what the source writes and what this node is identified by. */
+const EXTERNAL_ECOSYSTEM = "pypi";
+/** An external node is attributed from the file's own `import`/`from`
+ *  statements, which is module resolution and nothing more. R1 exactly: never
+ *  R0 (the import *is* the resolution), never R4 (nothing observed it run). */
+const EXTERNAL_RESOLUTION = 1;
 /**
  * Confidence by the level an edge's evidence earned — never by the level the
  * run reached. DEC-058: those were two different notions in the first adapter,
@@ -79,10 +88,6 @@ const REASONS = {
     callBaseUnresolved: "an HTTP call written as f\"{BASE}/path\" whose base this file does not bind to a string " +
         "literal. The path half is readable and the base half is not, so the served template cannot " +
         "be assembled — the named base is the piece of work that would close it.",
-    callBaseIsExternalHost: "an HTTP call written as f\"{BASE}/path\" whose base resolves to an absolute URL on another " +
-        "host. Read correctly and out of scope for the join: an endpoint is identified by method and " +
-        "path with no host (DEC-014), so claiming one here would let this caller join a local route " +
-        "of the same path that it never reaches.",
     callPathFromLocal: "an HTTP call whose path is a local assembled earlier in the function. Distinguished from an " +
         "opaque path because it is reachable by further work, and filing the two together would " +
         "report one number for two populations.",
@@ -354,6 +359,24 @@ export function extract(input) {
     const edges = [];
     const unresolved = [];
     const seenEdge = new Set();
+    /**
+     * DEC-164's `attrs.clientBase`, per caller node id. First writer wins, so a
+     * function making two calls is described by the first one read rather than
+     * by whichever happened to be walked last.
+     *
+     * **Declared HERE, beside `nodes`, and the reason is a bug this already
+     * caused.** It sat next to the R2 guard, which reads as the right place —
+     * the client half is R2 work. But the client half runs inside `emitRoutes`,
+     * and `emitRoutes` is *called* above that line, so every run threw
+     * `ReferenceError: Cannot access 'clientBases' before initialization`,
+     * `extractOrDegrade` turned the throw into a degraded one-node-per-file
+     * graph, and the corpus reported all nineteen patterns as role-unbound
+     * failures rather than as a crash. A temporal-dead-zone error inside a
+     * catch-all degrade path looks exactly like an adapter that read nothing.
+     * The map stays empty below R2 because nothing writes to it there, which is
+     * the honest version of what the old placement was trying to say.
+     */
+    const clientBases = new Map();
     /** file -> declaration name (first segment) -> Declared */
     const declaredIn = new Map();
     /** node id -> the entry owning it, where it sits in `nodes`, and how many
@@ -430,19 +453,26 @@ export function extract(input) {
         }));
         assignedOf.set(file, new Set(parsed.assignedNames ?? []));
     }
+    /** Node ids already minted for a dependency symbol — one node per symbol, not
+     *  per call site. Measured node growth is +3.1% (plane) to +4.5% (dispatch). */
+    const externalIds = new Map();
     /**
-     * The module and symbol behind a call that left the repository, as ledger
-     * `attrs`, or `undefined` when no binding the developer wrote settles it.
+     * The id of the `FUNCTION` node standing for the symbol behind a call that
+     * left the repository, or `undefined` when no binding the developer wrote
+     * settles it — or when this run did not reach the resolution such a node
+     * requires.
      *
-     * **This does not mint a node or an edge.** `@descryy/ir` gained the
-     * discriminator that lets a `FUNCTION` stand for a symbol in a dependency
-     * only at 0.7.0; this package is pinned to the release before it, and a node
-     * emitted against that release is rejected with every edge to it — losing the
-     * ledger row as well, which is strictly worse than the boundary being
-     * unnamed. The attribution is computed at the site that files the row so that
-     * minting is the only step left.
+     * Three settled rules govern the node
+     * (`DEC-NEXT-external-symbol-nodes-at-the-dependency-boundary`): the qsp
+     * carries an `ext:` marker (`externalQsp`), the node may claim R1–R3 but
+     * never R0 or R4, and it owns no outgoing edge, because nothing read its
+     * body. `thirdParty` rides in `attrs`, never in the hash: a module that moved
+     * between the standard library and an installed package would otherwise
+     * change identity with no line of source changing.
      */
-    const externalAttrs = (file, receiver, name) => {
+    const mintExternal = (file, receiver, name) => {
+        if (input.reached < EXTERNAL_RESOLUTION)
+            return undefined;
         const attribution = attributeExternal({
             receiver,
             name,
@@ -451,18 +481,49 @@ export function extract(input) {
         });
         if (attribution === null)
             return undefined;
-        return {
-            external: {
-                // Identity, and it is the module path the source imported — never a
-                // site-packages directory, never a distribution name, never a version.
-                moduleOrNamespace: attribution.moduleOrNamespace,
-                symbol: attribution.symbolPath.join("."),
+        // Identity, and it is the module path the source imported — never a
+        // site-packages directory, never a distribution name, never a version.
+        const qsp = externalQsp(attribution.moduleOrNamespace, attribution.symbolPath);
+        const existing = externalIds.get(qsp);
+        if (existing !== undefined)
+            return existing;
+        const id = nodeId(scope, "FUNCTION", qsp, LANGUAGE);
+        externalIds.set(qsp, id);
+        nodes.push({
+            id,
+            type: "FUNCTION",
+            name: attribution.symbolPath[attribution.symbolPath.length - 1] ?? "",
+            // No file and no range: the symbol lives in a dependency, and a node that
+            // named a file here would be invalidated by a source file it has nothing
+            // to do with.
+            file: null,
+            range: null,
+            language: LANGUAGE,
+            producedBy: input.producedBy,
+            resolution: EXTERNAL_RESOLUTION,
+            attrs: {
                 thirdParty: attribution.thirdParty,
                 // Honest degradation: `thirdParty` is decided against a static list,
-                // not against anything installed, and the row says which list.
+                // not against anything installed, and the node says which list.
                 thirdPartyBasis: STDLIB_BASIS,
             },
-        };
+            external: {
+                ecosystem: EXTERNAL_ECOSYSTEM,
+                moduleOrNamespace: attribution.moduleOrNamespace,
+            },
+        });
+        return id;
+    };
+    /**
+     * The edge into a dependency is **always `CALLS`, never `TESTS`**, even when
+     * the container is a `TEST_CASE` and every in-repo edge it owns is a `TESTS`.
+     * `TESTS` asserts coverage — that this case exercises that node — and this
+     * run read nothing of the dependency's body, so it cannot assert it (rule 4,
+     * rule 7). A test that reaches `yaml.safe_load` calls it; it does not test
+     * it. Filed as `DEC-NEXT-external-edge-is-calls-not-tests`.
+     */
+    const callExternal = (from, to) => {
+        push({ from, to, type: "CALLS" }, EXTERNAL_RESOLUTION);
     };
     // --- which classes are test suites, decided from evidence not from names ---
     /** file -> top-level class name -> the bases it was written with. */
@@ -1477,7 +1538,12 @@ export function extract(input) {
                     // left into is readable. This is the half `adapter-go` has no
                     // analogue for (Go has no from-import), and on measurement it is the
                     // larger half: 2,790 of 4,836 attributions on dispatch.
-                    disclose(source, edgeType, reference.name, file, reference.line, "outside", externalAttrs(file, undefined, reference.name));
+                    const external = mintExternal(file, undefined, reference.name);
+                    if (external !== undefined) {
+                        callExternal(source.id, external);
+                        continue;
+                    }
+                    disclose(source, edgeType, reference.name, file, reference.line, "outside");
                     continue;
                 }
                 if ((target.type === "FUNCTION" || target.type === "TEST_CASE") && target.id !== source.id) {
@@ -1560,8 +1626,16 @@ export function extract(input) {
                     // about this row — there is one, and it leaves the repository. So the
                     // reason becomes `outside`, which is what the row actually is. An
                     // unattributed row is untouched and stays `unknown`.
-                    const external = externalAttrs(file, reference.receiver, reference.name);
-                    disclose(source, edgeType, `${reference.receiver}.${reference.name}`, file, reference.line, external === undefined ? "unknown" : "outside", external);
+                    const external = mintExternal(file, reference.receiver, reference.name);
+                    if (external !== undefined) {
+                        // `import yaml` ... `yaml.safe_load(raw)`. When an import statement
+                        // binds the receiver, "no resolvable binding" is a FALSE sentence
+                        // about this row — there is one, and it leaves the repository. It
+                        // is now a node and an edge rather than a better-worded refusal.
+                        callExternal(source.id, external);
+                        continue;
+                    }
+                    disclose(source, edgeType, `${reference.receiver}.${reference.name}`, file, reference.line, "unknown");
                     continue;
                 }
                 const method = byFullName
@@ -1781,6 +1855,11 @@ export function extract(input) {
         const mounts = [];
         const routeDecls = [];
         const clientCalls = [];
+        /**
+         * Every file's client half, read but not yet judged — see the push site
+         * for why deciding is deferred to after the loop.
+         */
+        const pendingClients = [];
         const djangoRouteEntries = [];
         const djangoIncludeEntries = [];
         const drfRegisterEntries = [];
@@ -1849,27 +1928,22 @@ export function extract(input) {
                 const moduleId = moduleNodeId.get(file);
                 return moduleId === undefined ? undefined : { id: moduleId };
             };
-            for (const refusal of read.refusals) {
-                const container = containerFor([]);
-                if (container === undefined)
-                    continue;
-                disclose(container, "USES_API", refusal.rawTarget, refusal.file, refusal.line, refusal.reason, 
-                // Unset `refusalClass` stays legal and means unclassified — DEC-242 — so
-                // `attrs` is omitted entirely rather than sent with an `undefined` field.
-                refusal.refusalClass === undefined
-                    ? undefined
-                    : {
-                        blockedBy: refusal.blockedBy,
-                        refusalClass: refusal.refusalClass,
-                        ...(refusal.argumentKind === undefined ? {} : { argumentKind: refusal.argumentKind }),
-                    });
-            }
-            for (const call of read.calls) {
-                const container = containerFor(call.scope);
-                if (container === undefined)
-                    continue;
-                clientCalls.push({ call, containerId: container.id });
-            }
+            // EMITTED AFTER THE LOOP, NOT HERE — DEC-164's origin rule needs every
+            // file's bases before any one call site can be judged, and a base three
+            // files away is still evidence about this call. Reading is per file;
+            // deciding is per program. File order is preserved, so the ledger reads
+            // the same as it did.
+            pendingClients.push({
+                file,
+                calls,
+                moduleOfImport: moduleOfImportIn(file),
+                clientLocals,
+                constants,
+                localStrings,
+                functionParams,
+                containerFor,
+                read,
+            });
             for (const declaration of parsed.declarations) {
                 for (const call of declaration.decoratorCalls ?? []) {
                     const route = routeFromDecorator(file, call, declaration.path);
@@ -1893,6 +1967,81 @@ export function extract(input) {
                     drfRegisterEntries.push(reg);
             }
         }
+        // --- the client half, decided now that every file has been read --------
+        //
+        // DEC-164's own-origin rule admits an absolute call path only at an origin
+        // some base in this same program already resolves to. The first read
+        // produced those origins; the second re-reads only the files that could
+        // change, and with no absolute base anywhere `origins` is empty and no
+        // file is re-read at all.
+        const origins = new Set();
+        for (const pending of pendingClients) {
+            for (const call of pending.read.calls) {
+                if (call.baseOrigin !== undefined)
+                    origins.add(call.baseOrigin);
+            }
+        }
+        for (const pending of pendingClients) {
+            const couldChange = origins.size > 0 && pending.read.refusals.some((refusal) => refusal.reason === "callPathNotRelative");
+            const read = couldChange
+                ? clientCallsIn(pending.file, pending.calls, pending.moduleOfImport, pending.clientLocals, pending.constants, pending.localStrings, pending.functionParams, origins)
+                : pending.read;
+            for (const refusal of read.refusals) {
+                // DEC-164 is attributed to the caller that wrote the call; the ledger
+                // row's own container is unchanged and still the module.
+                const owner = pending.containerFor(refusal.scope);
+                if (owner !== undefined && refusal.clientBase !== undefined && !clientBases.has(owner.id)) {
+                    clientBases.set(owner.id, refusal.clientBase);
+                }
+                const container = pending.containerFor([]);
+                if (container === undefined)
+                    continue;
+                disclose(container, "USES_API", refusal.rawTarget, refusal.file, refusal.line, refusal.reason, 
+                // Unset `refusalClass` stays legal and means unclassified — DEC-242 — so
+                // `attrs` is omitted entirely rather than sent with an `undefined` field.
+                refusal.refusalClass === undefined
+                    ? undefined
+                    : {
+                        blockedBy: refusal.blockedBy,
+                        refusalClass: refusal.refusalClass,
+                        ...(refusal.argumentKind === undefined ? {} : { argumentKind: refusal.argumentKind }),
+                    });
+            }
+            // Pydantic parses written in this file, read once rather than per call
+            // — golden 18's consumer half. Read HERE and not at the first pass
+            // because `read` may be the SECOND read of this file (DEC-164's origin
+            // rule re-reads a file whose refusals could change), and a model
+            // resolved against the first read's calls would describe calls that no
+            // longer exist.
+            const parses = validateCalls(pending.calls);
+            for (const call of read.calls) {
+                const container = pending.containerFor(call.scope);
+                if (container === undefined)
+                    continue;
+                if (!clientBases.has(container.id))
+                    clientBases.set(container.id, call.clientBase);
+                clientCalls.push({
+                    call,
+                    containerId: container.id,
+                    responseModel: modelParsing(parses, call.scope, call.client, call.rawPath),
+                    responseModelFile: pending.file,
+                });
+            }
+        }
+        // DEC-164, patched onto the caller now rather than at the end of the
+        // function. `adapter-csharp` patches its own at the end and can, because
+        // it has one exit; this function returns early at `reached < 3`, and a
+        // patch after that line was **absent at R2 while the golden requires it
+        // there** — the conformance sweep caught it as three `node-attr-mismatch`
+        // findings at R2 only, which a single-level run would never have shown.
+        // Here is the earliest point where every caller node exists and every
+        // base has been decided.
+        for (let i = 0; i < nodes.length; i += 1) {
+            const base = clientBases.get(nodes[i].id);
+            if (base === undefined)
+                continue;
+            nodes[i] = { ...nodes[i], attrs: { ...nodes[i].attrs, clientBase: base } };
+        }
         /** child -> its parent and the prefix mounting it there. */
         const parentOf = new Map();
         for (const mount of mounts) {
@@ -2664,7 +2813,9 @@ export function extract(input) {
         // rejected at the boundary as RESOLUTION_EXCEEDS_BATCH — the batch
         // contract catching a claim stronger than the run that produced it, which
         // is exactly what it is for.
-        for (const { call, containerId } of input.reached >= 2 ? clientCalls : []) {
+        for (const { call, containerId, responseModel, responseModelFile } of input.reached >= 2
+            ? clientCalls
+            : []) {
             const endpointId = nodeId(scope, "API_ENDPOINT", endpointQsp(call.method, call.path), null);
             if (!routeSeen.has(endpointId)) {
                 routeSeen.add(endpointId);
@@ -2680,6 +2831,26 @@ export function extract(input) {
                     attrs: { method: call.method, pathTemplate: normaliseEndpointPath(call.path) },
                 });
             }
+            /**
+             * Golden 18's consumer half. Gated on `input.reached >= 3` INSIDE the
+             * attrs rather than by an early return above: this loop runs from R2, so
+             * a guard placed one level up would be green at R3 and silently wrong at
+             * R2 — the monotonic sweep's own failure mode. At R2 the key is simply
+             * absent, which is the honest answer.
+             *
+             * R3 because the shape is a Pydantic DECLARATION — types stated, not
+             * inferred — which is DEC-202's rule and the same ground `adapter-java`'s
+             * JPA shapes and `adapter-sql`'s schema stand on. The edge itself stays
+             * at 2 (path resolution); `shapeResolution` carries the shape's own
+             * evidence so the contract engine does not cap it at the weaker of the
+             * two facts this edge holds (DEC-058).
+             */
+            const declaredShape = input.reached >= 3 && responseModel !== null
+                ? followName(responseModelFile, responseModel)
+                : undefined;
+            const shapeAttrs = declaredShape !== undefined && declaredShape.type === "DTO"
+                ? { responseType: declaredShape.id, shapeResolution: 3 }
+                : {};
             push({
                 from: containerId,
                 to: endpointId,
@@ -2689,6 +2860,7 @@ export function extract(input) {
                     file: call.file,
                     line: call.line,
                     written: call.rawPath,
+                    ...shapeAttrs,
                 },
             }, 2);
         }