@neat.is/core 0.10.3 → 0.10.5-dev.20261001

package/dist/index.d.cts

@@ -45,6 +45,7 @@ interface ExtractOptions {
     onPolicyTrigger?: (graph: NeatGraph) => Promise<void> | void;
     project?: string;
     errorsPath?: string;
+    source?: string;
 }
 declare function extractFromDirectory(graph: NeatGraph, scanPath: string, opts?: ExtractOptions): Promise<ExtractResult>;
 
@@ -126,6 +127,29 @@ declare class Projects {
     attachSearchIndex(name: string, index: SearchIndex | undefined): void;
 }
 
+/** What one pass did: whether the bound-repo list was read, and how the repos it took on came out. */
+interface RepoSyncPassCounts {
+    listed: boolean;
+    synced: number;
+    failed: number;
+}
+/** A finished pass, as the on-demand trigger reports it. */
+interface RepoSyncPassSummary extends RepoSyncPassCounts {
+    startedAt: string;
+    finishedAt: string;
+}
+/**
+ * The answer to "sync now" (#1293). `started` — nothing was running, a pass has begun. `queued` — a pass was
+ * already running, and one more will follow it. Joining the running pass outright would be wrong: it read
+ * the bound-repo list before this request, so a repo bound since would be missed; the follow-up re-reads the
+ * list. Any number of requests during one pass coalesce into that single follow-up. `lastPass` is the most
+ * recent pass to finish, absent until one has.
+ */
+interface RepoSyncRequestResult {
+    status: 'started' | 'queued';
+    lastPass?: RepoSyncPassSummary;
+}
+
 /**
  * A connector implements exactly one method. Everything downstream of
  * `poll()` — resolving a static call site, minting the OBSERVED edge — is
@@ -293,6 +317,7 @@ interface BuildApiOptions {
     authToken?: string;
     trustProxy?: boolean;
     publicRead?: boolean;
+    otlpReceiver?: () => FastifyInstance | undefined;
     bootstrap?: {
         status: (name: string) => 'bootstrapping' | 'active' | 'broken' | undefined;
         list: () => Array<{
@@ -306,6 +331,7 @@ interface BuildApiOptions {
         path: string;
     };
     connectorsHome?: string;
+    repoSync?: (project: string) => (() => RepoSyncRequestResult) | undefined;
     runPoll?: typeof runConnectorPoll;
 }
 declare function buildApi(opts: BuildApiOptions): Promise<FastifyInstance>;
@@ -597,6 +623,7 @@ interface ProjectSlot {
     stopHostedConnectors: () => void;
     stopK8sSubstrate: () => void;
     stopRepoSync: () => void;
+    syncRepos?: () => RepoSyncRequestResult;
     detachEvents: () => void;
     status: 'active' | 'broken';
     errorReason?: string;

package/dist/index.d.ts

@@ -45,6 +45,7 @@ interface ExtractOptions {
     onPolicyTrigger?: (graph: NeatGraph) => Promise<void> | void;
     project?: string;
     errorsPath?: string;
+    source?: string;
 }
 declare function extractFromDirectory(graph: NeatGraph, scanPath: string, opts?: ExtractOptions): Promise<ExtractResult>;
 
@@ -126,6 +127,29 @@ declare class Projects {
     attachSearchIndex(name: string, index: SearchIndex | undefined): void;
 }
 
+/** What one pass did: whether the bound-repo list was read, and how the repos it took on came out. */
+interface RepoSyncPassCounts {
+    listed: boolean;
+    synced: number;
+    failed: number;
+}
+/** A finished pass, as the on-demand trigger reports it. */
+interface RepoSyncPassSummary extends RepoSyncPassCounts {
+    startedAt: string;
+    finishedAt: string;
+}
+/**
+ * The answer to "sync now" (#1293). `started` — nothing was running, a pass has begun. `queued` — a pass was
+ * already running, and one more will follow it. Joining the running pass outright would be wrong: it read
+ * the bound-repo list before this request, so a repo bound since would be missed; the follow-up re-reads the
+ * list. Any number of requests during one pass coalesce into that single follow-up. `lastPass` is the most
+ * recent pass to finish, absent until one has.
+ */
+interface RepoSyncRequestResult {
+    status: 'started' | 'queued';
+    lastPass?: RepoSyncPassSummary;
+}
+
 /**
  * A connector implements exactly one method. Everything downstream of
  * `poll()` — resolving a static call site, minting the OBSERVED edge — is
@@ -293,6 +317,7 @@ interface BuildApiOptions {
     authToken?: string;
     trustProxy?: boolean;
     publicRead?: boolean;
+    otlpReceiver?: () => FastifyInstance | undefined;
     bootstrap?: {
         status: (name: string) => 'bootstrapping' | 'active' | 'broken' | undefined;
         list: () => Array<{
@@ -306,6 +331,7 @@ interface BuildApiOptions {
         path: string;
     };
     connectorsHome?: string;
+    repoSync?: (project: string) => (() => RepoSyncRequestResult) | undefined;
     runPoll?: typeof runConnectorPoll;
 }
 declare function buildApi(opts: BuildApiOptions): Promise<FastifyInstance>;
@@ -597,6 +623,7 @@ interface ProjectSlot {
     stopHostedConnectors: () => void;
     stopK8sSubstrate: () => void;
     stopRepoSync: () => void;
+    syncRepos?: () => RepoSyncRequestResult;
     detachEvents: () => void;
     status: 'active' | 'broken';
     errorReason?: string;

package/dist/index.js

@@ -1,7 +1,7 @@
 import {
   routeSpanToProject,
   startDaemon
-} from "./chunk-DLQNMU3R.js";
+} from "./chunk-TONL5O2E.js";
 import {
   ProjectNameCollisionError,
   addProject,
@@ -37,15 +37,15 @@ import {
   thresholdForEdgeType,
   touchLastSeen,
   writeAtomically
-} from "./chunk-LNIVRE4L.js";
+} from "./chunk-HHAPD6JD.js";
 import {
   startOtelGrpcReceiver
-} from "./chunk-5SJHA5ZH.js";
+} from "./chunk-LXQCL4MP.js";
 import {
   buildOtelReceiver,
   logSpanHandler,
   parseOtlpRequest
-} from "./chunk-M7CGATVC.js";
+} from "./chunk-EFBWQIKG.js";
 export {
   ProjectNameCollisionError,
   addProject,

package/dist/neatd.cjs

@@ -51,6 +51,10 @@ function assertBindAuthority(host, token) {
   if (isLoopbackHost2(host)) return;
   throw new BindAuthorityError(host);
 }
+function isBearerDelegated(req2) {
+  const config = req2.routeOptions?.config;
+  return config?.bearerDelegated === true;
+}
 function mountBearerAuth(app, opts) {
   if (!opts.token || opts.token.length === 0) return;
   if (opts.trustProxy) return;
@@ -66,6 +70,10 @@ function mountBearerAuth(app, opts) {
       done();
       return;
     }
+    if (isBearerDelegated(req2)) {
+      done();
+      return;
+    }
     if (publicRead && PUBLIC_READ_METHODS.has(req2.method)) {
       done();
       return;
@@ -99,7 +107,7 @@ function readAuthEnv(env = process.env) {
     publicRead: parseBoolEnv(env.NEAT_PUBLIC_READ)
   };
 }
-var import_node_crypto, LOOPBACK_HOSTS, BindAuthorityError, PUBLIC_READ_METHODS, DEFAULT_UNAUTH_PATHS, PROJECT_SCOPED_UNAUTH_PATTERN;
+var import_node_crypto, LOOPBACK_HOSTS, BindAuthorityError, PUBLIC_READ_METHODS, DEFAULT_UNAUTH_PATHS, PROJECT_SCOPED_UNAUTH_PATTERN, BEARER_DELEGATED;
 var init_auth = __esm({
   "src/auth.ts"() {
     "use strict";
@@ -129,6 +137,7 @@ var init_auth = __esm({
     PROJECT_SCOPED_UNAUTH_PATTERN = new RegExp(
       `^/projects/[^/]+/(?:${DEFAULT_UNAUTH_PATHS.map((p) => p.slice(1)).join("|")})$`
     );
+    BEARER_DELEGATED = { bearerDelegated: true };
   }
 });
 
@@ -3353,6 +3362,10 @@ function languageForPath(relPath) {
       return void 0;
   }
 }
+var currentSource;
+function setExtractionSource(source) {
+  currentSource = source;
+}
 function ensureFileNode(graph, serviceName, serviceNodeId, relPath) {
   let nodesAdded = 0;
   let edgesAdded = 0;
@@ -3365,10 +3378,16 @@ function ensureFileNode(graph, serviceName, serviceNodeId, relPath) {
       service: serviceName,
       path: relPath,
       ...language ? { language } : {},
+      ...currentSource ? { source: currentSource } : {},
       discoveredVia: "static"
     };
     graph.addNode(fileNodeId, node);
     nodesAdded++;
+  } else if (currentSource) {
+    const existing = graph.getNodeAttributes(fileNodeId);
+    if (existing.source !== currentSource) {
+      graph.replaceNodeAttributes(fileNodeId, { ...existing, source: currentSource });
+    }
   }
   const containsId = (0, import_types4.extractedEdgeId)(serviceNodeId, fileNodeId, import_types4.EdgeType.CONTAINS);
   if (!graph.hasEdge(containsId)) {
@@ -16426,10 +16445,12 @@ init_cjs_shims();
 var import_node_fs33 = require("fs");
 var import_node_path68 = __toESM(require("path"), 1);
 var import_types58 = require("@neat.is/types");
-function dropOrphanedFileNodes(graph) {
+function dropOrphanedFileNodes(graph, source) {
   const orphans = [];
   graph.forEachNode((id, attrs) => {
-    if (attrs.type !== import_types58.NodeType.FileNode) return;
+    const node = attrs;
+    if (node.type !== import_types58.NodeType.FileNode) return;
+    if (source && node.source !== source) return;
     if (graph.inboundEdges(id).length === 0 && graph.outboundEdges(id).length === 0) {
       orphans.push(id);
     }
@@ -16437,12 +16458,23 @@ function dropOrphanedFileNodes(graph) {
   for (const id of orphans) graph.dropNode(id);
   return orphans.length;
 }
-function retireExtractedEdgesByMissingFile(graph, scanPath, serviceDirs = []) {
+function outOfScope(graph, edge, source) {
+  if (!source) return false;
+  for (const endpoint2 of [edge.source, edge.target]) {
+    if (!graph.hasNode(endpoint2)) continue;
+    const node = graph.getNodeAttributes(endpoint2);
+    if (node.type !== import_types58.NodeType.FileNode) continue;
+    return node.source !== source;
+  }
+  return false;
+}
+function retireExtractedEdgesByMissingFile(graph, scanPath, serviceDirs = [], source) {
   const toDrop = [];
   const bases = [scanPath, ...serviceDirs];
   graph.forEachEdge((id, attrs) => {
     const edge = attrs;
     if (edge.provenance !== import_types58.Provenance.EXTRACTED) return;
+    if (outOfScope(graph, edge, source)) return;
     const evidenceFile = edge.evidence?.file;
     if (!evidenceFile) return;
     if (import_node_path68.default.isAbsolute(evidenceFile)) {
@@ -16453,7 +16485,7 @@ function retireExtractedEdgesByMissingFile(graph, scanPath, serviceDirs = []) {
     if (!found) toDrop.push(id);
   });
   for (const id of toDrop) graph.dropEdge(id);
-  dropOrphanedFileNodes(graph);
+  dropOrphanedFileNodes(graph, source);
   return toDrop.length;
 }
 
@@ -16462,6 +16494,7 @@ async function extractFromDirectory(graph, scanPath, opts = {}) {
   await ensureCompatLoaded();
   drainExtractionErrors();
   drainSkippedFiles();
+  setExtractionSource(opts.source);
   const services = await discoverServices(scanPath);
   const phase1Nodes = addServiceNodes(graph, services);
   await addServiceAliases(graph, scanPath, services);
@@ -16482,7 +16515,8 @@ async function extractFromDirectory(graph, scanPath, opts = {}) {
   const ghostsRetired = retireExtractedEdgesByMissingFile(
     graph,
     scanPath,
-    services.map((s) => s.dir)
+    services.map((s) => s.dir),
+    opts.source
   );
   const frontiersPromoted = promoteFrontierNodes(graph);
   if (opts.onPolicyTrigger) await opts.onPolicyTrigger(graph);
@@ -16517,6 +16551,7 @@ async function extractFromDirectory(graph, scanPath, opts = {}) {
       );
     }
   }
+  setExtractionSource(void 0);
   const result = {
     nodesAdded: phase1Nodes + fileEnum.nodesAdded + symbolEnum.nodesAdded + importGraph.nodesAdded + symbolEdges.nodesAdded + actionPhase.nodesAdded + phase2.nodesAdded + phase3.nodesAdded + routePhase.nodesAdded + grpcPhase.nodesAdded + phase4.nodesAdded + tableEdges.nodesAdded + phase5.nodesAdded + zodPhase.nodesAdded + firestoreRulesPhase.nodesAdded,
     edgesAdded: fileEnum.edgesAdded + symbolEnum.edgesAdded + importGraph.edgesAdded + symbolEdges.edgesAdded + actionPhase.edgesAdded + phase2.edgesAdded + phase3.edgesAdded + routePhase.edgesAdded + grpcPhase.edgesAdded + phase4.edgesAdded + tableEdges.edgesAdded + phase5.edgesAdded + zodPhase.edgesAdded + firestoreRulesPhase.edgesAdded,
@@ -23518,6 +23553,19 @@ function registerRoutes(scope, ctx) {
       });
     }
   });
+  scope.post("/repo-sync", async (req2, reply) => {
+    const proj = resolveProject(registry, req2, reply, ctx.bootstrap, ctx.singleProject);
+    if (!proj) return;
+    const syncNow = ctx.repoSync?.(proj.name);
+    if (!syncNow) {
+      return reply.code(404).send({
+        error: "repo-sync is not running for this project",
+        project: proj.name,
+        hint: "Only a hosted daemon syncs bound repos. On a local daemon, POST /graph/scan re-extracts the project path."
+      });
+    }
+    return reply.code(202).send({ project: proj.name, ...syncNow() });
+  });
   scope.post("/graph/scan", async (req2, reply) => {
     const proj = resolveProject(registry, req2, reply, ctx.bootstrap, ctx.singleProject);
     if (!proj) return;
@@ -23765,6 +23813,39 @@ function registerRoutes(scope, ctx) {
     }
   });
 }
+var OTLP_BODY_LIMIT = 16 * 1024 * 1024;
+function otlpOnRest(getReceiver) {
+  return async function plugin(scope) {
+    scope.removeAllContentTypeParsers();
+    scope.addContentTypeParser(
+      "*",
+      { parseAs: "buffer", bodyLimit: OTLP_BODY_LIMIT },
+      (_req, body, done) => done(null, body)
+    );
+    const relay = async (req2, reply) => {
+      const receiver = getReceiver();
+      if (!receiver) {
+        return reply.code(503).header("retry-after", "1").send({ error: "OTLP receiver is starting" });
+      }
+      const headers = { ...req2.headers };
+      delete headers["content-length"];
+      delete headers["transfer-encoding"];
+      delete headers.connection;
+      const res = await receiver.inject({
+        method: "POST",
+        url: req2.url,
+        headers,
+        payload: req2.body
+      });
+      const contentType = res.headers["content-type"];
+      if (contentType !== void 0) void reply.header("content-type", contentType);
+      return reply.code(res.statusCode).send(res.rawPayload);
+    };
+    const route = { config: BEARER_DELEGATED, bodyLimit: OTLP_BODY_LIMIT };
+    scope.post("/v1/traces", route, relay);
+    scope.post("/projects/:project/v1/traces", route, relay);
+  };
+}
 async function buildApi(opts) {
   const app = (0, import_fastify2.default)({ logger: false, routerOptions: { maxParamLength: 1024 } });
   await app.register(import_cors.default, { origin: true });
@@ -23783,6 +23864,7 @@ async function buildApi(opts) {
     authProxy: trustProxy === true,
     requiresAuth
   }));
+  if (opts.otlpReceiver) await app.register(otlpOnRest(opts.otlpReceiver));
   const startedAt = opts.startedAt ?? Date.now();
   const registry = buildLegacyRegistry(opts);
   const legacyErrorsExplicit = !opts.projects && opts.errorsPath !== void 0;
@@ -23813,6 +23895,7 @@ async function buildApi(opts) {
     bootstrap: opts.bootstrap,
     singleProject: opts.singleProject?.name,
     connectorsHome: opts.connectorsHome,
+    repoSync: opts.repoSync,
     runPoll: opts.runPoll ?? runConnectorPoll
   };
   app.get("/health", async () => {
@@ -24363,7 +24446,7 @@ async function syncOneRepo(r, input) {
     await cpPostStatus(deps, r.owner, r.name, { syncStatus: "syncing", detail: "cloning" }).catch(() => {
     });
     await cloneRepo(r.cloneUrl, r.defaultBranch, dir);
-    const extracted = await extract(graph, dir);
+    const extracted = await extract(graph, dir, { source: label });
     const nodes = extracted?.nodesAdded ?? 0;
     const edges = extracted?.edgesAdded ?? 0;
     await cpPostStatus(deps, r.owner, r.name, {
@@ -24371,6 +24454,7 @@ async function syncOneRepo(r, input) {
       detail: `extracted ${nodes} node${nodes === 1 ? "" : "s"}, ${edges} edge${edges === 1 ? "" : "s"}`,
       lastSyncAt: new Date(input.now?.() ?? Date.now()).toISOString()
     });
+    return true;
   } catch (err) {
     input.onError?.(label, err);
     await cpPostStatus(deps, r.owner, r.name, {
@@ -24378,50 +24462,76 @@ async function syncOneRepo(r, input) {
       detail: scrubToken(err.message).slice(0, 300)
     }).catch(() => {
     });
+    return false;
   } finally {
     if (dir) await (0, import_promises2.rm)(dir, { recursive: true, force: true }).catch(() => {
     });
   }
 }
-async function runRepoSyncPass(input) {
+async function runRepoSyncPassCounted(input) {
+  const counts = { listed: false, synced: 0, failed: 0 };
   let repos;
   try {
     repos = await cpGet2(`/internal/projects/${input.deps.projectId}/repos`, input.deps);
   } catch (err) {
     input.onSkip?.("(all)", `control plane repo list unreadable \u2014 ${err.message}`);
-    return false;
+    return counts;
   }
-  if (!Array.isArray(repos)) return false;
+  if (!Array.isArray(repos)) return counts;
+  counts.listed = true;
   for (const r of repos) {
     if (!input.forceResync && !needsSync(r)) continue;
-    await syncOneRepo(r, input);
+    if (await syncOneRepo(r, input)) counts.synced++;
+    else counts.failed++;
   }
-  return true;
+  return counts;
 }
 async function startRepoSync(input) {
   const intervalMs = input.intervalMs ?? DEFAULT_SYNC_INTERVAL_MS;
+  const now = () => input.now?.() ?? Date.now();
   let stopped = false;
-  let running = false;
+  let inFlight = null;
+  let again = false;
+  let lastPass;
   let bootResyncDone = false;
-  const tick = async () => {
-    if (stopped || running) return;
-    running = true;
+  const run = async () => {
     try {
-      const listed = await runRepoSyncPass({ ...input, forceResync: !bootResyncDone });
-      if (listed) bootResyncDone = true;
+      do {
+        again = false;
+        const startedAt = new Date(now()).toISOString();
+        const counts = await runRepoSyncPassCounted({ ...input, forceResync: !bootResyncDone });
+        if (counts.listed) bootResyncDone = true;
+        lastPass = { ...counts, startedAt, finishedAt: new Date(now()).toISOString() };
+      } while (again && !stopped);
     } finally {
-      running = false;
+      inFlight = null;
     }
   };
-  void tick();
-  const timer = setInterval(() => {
-    void tick();
-  }, intervalMs);
+  const tick = () => {
+    if (stopped || inFlight) return;
+    inFlight = run();
+  };
+  tick();
+  const timer = setInterval(tick, intervalMs);
   if (typeof timer.unref === "function") timer.unref();
-  return () => {
+  const stop = () => {
     stopped = true;
     clearInterval(timer);
   };
+  return Object.assign(stop, {
+    syncNow: () => {
+      const summary = lastPass ? { lastPass } : {};
+      if (inFlight) {
+        again = true;
+        return { status: "queued", ...summary };
+      }
+      inFlight = run();
+      return { status: "started", ...summary };
+    },
+    settled: async () => {
+      while (inFlight) await inFlight;
+    }
+  });
 }
 async function maybeStartRepoSync(input) {
   const env = input.env ?? process.env;
@@ -25133,7 +25243,7 @@ async function bootstrapProject(entry2, connectors = [], neatHome4) {
   await loadGraphFromDisk(graph, outPath);
   const detachEvents = attachGraphToEventBus(graph, { project: entry2.name });
   try {
-    await extractFromDirectory(graph, entry2.path);
+    await extractFromDirectory(graph, entry2.path, { source: entry2.name });
     const stopPersist = startPersistLoop(graph, outPath, { exitOnSignal: false });
     const stopStaleness = startStalenessLoop(graph, {
       staleEventsPath: paths.staleEventsPath,
@@ -25191,6 +25301,7 @@ async function bootstrapProject(entry2, connectors = [], neatHome4) {
       stopHostedConnectors,
       stopK8sSubstrate,
       stopRepoSync,
+      ...stopRepoSync.syncNow ? { syncRepos: stopRepoSync.syncNow } : {},
       detachEvents,
       status: "active"
     };
@@ -25462,7 +25573,14 @@ async function startDaemon(opts = {}) {
         // through the same resolved home the slot bootstrap read it from, so a
         // daemon given an explicit NEAT_HOME serves status for the same file it
         // polls.
-        connectorsHome: home
+        connectorsHome: home,
+        // #1293 — "sync this project's repos now". Looked up per request: the
+        // slot, and with it the trigger, appears once bootstrap has run.
+        repoSync: (name) => slots.get(name)?.syncRepos,
+        // OTLP answers on this listener too, so a host that routes a single
+        // port still delivers a tenant app's spans. Bound late: the receiver
+        // is built below, once the REST bind has succeeded.
+        otlpReceiver: () => otlpApp ?? void 0
       });
       restAddress = await restApp.listen({ port: restPort, host });
       console.log(
@@ -25643,7 +25761,9 @@ async function startDaemon(opts = {}) {
         }
       });
       otlpAddress = await listenSteppingOtlp(otlpApp, otlpPort, host);
-      console.log(`neatd: OTLP listening on ${otlpAddress}/v1/traces`);
+      console.log(
+        `neatd: OTLP listening on http://${host}:${portFromListenAddress(otlpAddress, otlpPort)}/v1/traces (also at /v1/traces on the REST port)`
+      );
     } catch (err) {
       for (const slot of slots.values()) {
         teardownSlot(slot);