@opentrace/components 0.1.1-rc.112 → 0.1.1-rc.120

package/dist/pipeline.cjs

@@ -3315,14 +3315,14 @@ class TemplateSummarizer {
   async dispose() {
   }
 }
-const TYPE_TO_KIND = {
+const TYPE_TO_KIND$1 = {
   Function: "function",
   Class: "class",
   File: "file",
   Directory: "directory"
 };
 function summarizeNode(node) {
-  const kind = TYPE_TO_KIND[node.type];
+  const kind = TYPE_TO_KIND$1[node.type];
   if (!kind) {
     return `${node.type} ${node.name}`;
   }
@@ -3431,8 +3431,537 @@ const DEFAULT_SUMMARIZER_CONFIG = {
   maxInputLength: 480,
   minLines: 5
 };
+function pushAll(dst, src) {
+  for (let j = 0; j < src.length; j++) {
+    dst.push(src[j]);
+  }
+}
+function* runNodePipeline(opts) {
+  const { ctx, stages, seeds } = opts;
+  const stageCount = stages.length;
+  const queues = Array.from({ length: stageCount }, () => []);
+  pushAll(queues[0], seeds);
+  let totalNodes = seeds.length;
+  let totalRelationships = 0;
+  const hasWork = () => queues.some((q) => q.length > 0);
+  while (hasWork()) {
+    if (ctx.cancelled) {
+      yield { kind: "pipeline_error", error: "cancelled" };
+      return;
+    }
+    let processed = false;
+    for (let i = stageCount - 1; i >= 0; i--) {
+      if (queues[i].length === 0) continue;
+      const node = queues[i].shift();
+      const stage = stages[i];
+      yield { stage: stage.name(), node: node.id, action: "start" };
+      let mutation;
+      try {
+        mutation = stage.process(node);
+      } catch (err) {
+        yield {
+          kind: "item_error",
+          stage: stage.name(),
+          node: node.id,
+          error: err instanceof Error ? err.message : String(err)
+        };
+        processed = true;
+        break;
+      }
+      yield { stage: stage.name(), node: node.id, action: "end", mutation };
+      totalRelationships += mutation.relationships.length;
+      if (mutation.nodes.length > 0) {
+        totalNodes += mutation.nodes.length;
+        if (i < stageCount - 1) {
+          pushAll(queues[i + 1], mutation.nodes);
+        }
+      }
+      processed = true;
+      break;
+    }
+    if (!processed) break;
+  }
+  for (const stage of stages) {
+    if (ctx.cancelled) {
+      yield { kind: "pipeline_error", error: "cancelled" };
+      return;
+    }
+    yield { kind: "flush_start", stage: stage.name() };
+    let mutation;
+    try {
+      mutation = stage.flush();
+    } catch (err) {
+      yield {
+        kind: "pipeline_error",
+        error: `flush error in ${stage.name()}: ${err instanceof Error ? err.message : String(err)}`
+      };
+      return;
+    }
+    totalNodes += mutation.nodes.length;
+    totalRelationships += mutation.relationships.length;
+    yield {
+      kind: "flush_end",
+      stage: stage.name(),
+      mutation: mutation.nodes.length > 0 || mutation.relationships.length > 0 ? mutation : void 0
+    };
+  }
+  yield {
+    kind: "pipeline_done",
+    totalNodes,
+    totalRelationships
+  };
+}
+const EMPTY_MUTATION = Object.freeze({
+  nodes: [],
+  relationships: []
+});
+const DEFAULT_CACHE_LIMIT = 500 * 1024 * 1024;
+class FileCacheStage {
+  cache = /* @__PURE__ */ new Map();
+  bytesUsed = 0;
+  byteLimit;
+  full = false;
+  cachedCount = 0;
+  skippedCount = 0;
+  constructor(config) {
+    this.byteLimit = config.byteLimit ?? DEFAULT_CACHE_LIMIT;
+    for (const [fileId, content] of config.fileContentMap) {
+      const byteSize = content.length * 2;
+      if (this.bytesUsed + byteSize <= this.byteLimit) {
+        this.cache.set(fileId, content);
+        this.bytesUsed += byteSize;
+        this.cachedCount++;
+      } else {
+        this.full = true;
+        this.skippedCount++;
+      }
+    }
+  }
+  name() {
+    return "cache";
+  }
+  process(node) {
+    return { nodes: [node], relationships: [] };
+  }
+  flush() {
+    return { nodes: [], relationships: [] };
+  }
+  /** Read cached content for a file. Returns undefined if not cached. */
+  getContent(fileId) {
+    return this.cache.get(fileId);
+  }
+  /**
+   * Remove a file from the raw cache (e.g. after extraction is done).
+   * Frees the JS string so GC can reclaim the memory.
+   */
+  evict(fileId) {
+    const content = this.cache.get(fileId);
+    if (content) {
+      this.bytesUsed -= content.length * 2;
+      this.cache.delete(fileId);
+    }
+  }
+  /** Current bytes used by the cache. */
+  getBytesUsed() {
+    return this.bytesUsed;
+  }
+  /** Whether the cache limit has been reached. */
+  isFull() {
+    return this.full;
+  }
+  /** Number of files cached vs skipped. */
+  stats() {
+    return {
+      cached: this.cachedCount,
+      skipped: this.skippedCount,
+      bytesUsed: this.bytesUsed,
+      byteLimit: this.byteLimit
+    };
+  }
+}
+class ExtractStage {
+  registries = {
+    nameRegistry: /* @__PURE__ */ new Map(),
+    fileRegistry: /* @__PURE__ */ new Map(),
+    classRegistry: /* @__PURE__ */ new Map(),
+    importRegistry: /* @__PURE__ */ new Map()
+  };
+  allCallInfo = [];
+  knownPaths;
+  pathToFileId;
+  goModulePath;
+  getContent;
+  packageNodes;
+  emittedNodeIds = /* @__PURE__ */ new Set();
+  pendingPackageNodes = [];
+  constructor(config) {
+    const { scanResult, getContent } = config;
+    this.knownPaths = scanResult.knownPaths;
+    this.pathToFileId = scanResult.pathToFileId;
+    this.goModulePath = scanResult.goModulePath;
+    this.getContent = getContent;
+    this.packageNodes = new Map(scanResult.packageNodes);
+  }
+  name() {
+    return "extract";
+  }
+  process(node) {
+    if (node.type !== "File") {
+      return { nodes: [node], relationships: [] };
+    }
+    const filePath = node.properties?.path;
+    if (!filePath) {
+      return { nodes: [node], relationships: [] };
+    }
+    const ext = getExtension(filePath);
+    const language = detectLanguage(ext);
+    if (!language) {
+      return { nodes: [node], relationships: [] };
+    }
+    const parser = getParserForLanguage(language, ext);
+    const extractor = getExtractor(language);
+    if (!parser || !extractor) {
+      return { nodes: [node], relationships: [] };
+    }
+    const fileId = node.id;
+    const content = this.getContent(fileId);
+    if (content === void 0) {
+      return { nodes: [node], relationships: [] };
+    }
+    const nodes = [node];
+    const rels = [];
+    try {
+      const tree = parser.parse(content);
+      if (!tree) {
+        return { nodes: [node], relationships: [] };
+      }
+      const extraction = extractor(tree.rootNode);
+      this.registries.fileRegistry.set(fileId, /* @__PURE__ */ new Map());
+      for (const sym of extraction.symbols) {
+        processSymbol(
+          sym,
+          fileId,
+          language,
+          this.registries,
+          this.allCallInfo,
+          nodes,
+          rels,
+          this.emittedNodeIds
+        );
+      }
+      const rootNode = extraction.rootNode;
+      if (rootNode) {
+        const importResult = analyzeImports(
+          rootNode,
+          language,
+          filePath,
+          this.knownPaths,
+          this.goModulePath
+        );
+        const fileImports = {};
+        const seenTargetFiles = /* @__PURE__ */ new Set();
+        for (const [alias, targetPath] of Object.entries(
+          importResult.internal
+        )) {
+          const targetFileId = this.pathToFileId.get(targetPath);
+          if (targetFileId) {
+            fileImports[alias] = targetFileId;
+            if (!seenTargetFiles.has(targetFileId)) {
+              seenTargetFiles.add(targetFileId);
+              rels.push({
+                id: `${fileId}->IMPORTS->${targetFileId}`,
+                type: "IMPORTS",
+                source_id: fileId,
+                target_id: targetFileId
+              });
+            }
+          }
+        }
+        this.registries.importRegistry.set(fileId, fileImports);
+        for (const [pkgName, pkgId] of Object.entries(
+          importResult.external
+        )) {
+          if (!this.packageNodes.has(pkgId)) {
+            const pkgNode = {
+              id: pkgId,
+              type: "Package",
+              name: pkgName,
+              properties: { registry: pkgId.split(":")[1] }
+            };
+            this.packageNodes.set(pkgId, pkgNode);
+            this.pendingPackageNodes.push(pkgNode);
+          }
+          rels.push({
+            id: `${fileId}->IMPORTS->${pkgId}`,
+            type: "IMPORTS",
+            source_id: fileId,
+            target_id: pkgId
+          });
+        }
+      }
+    } catch {
+      return { nodes: [node], relationships: [] };
+    }
+    return { nodes, relationships: rels };
+  }
+  flush() {
+    const nodes = this.pendingPackageNodes.splice(0);
+    return { nodes, relationships: [] };
+  }
+}
+class ResolveStage {
+  extractStage;
+  constructor(extractStage) {
+    this.extractStage = extractStage;
+  }
+  name() {
+    return "resolve";
+  }
+  process(node) {
+    return { nodes: [node], relationships: [] };
+  }
+  flush() {
+    const { registries, allCallInfo } = this.extractStage;
+    const resolvedCalls = resolveCalls(allCallInfo, registries);
+    const callRels = resolvedCallsToRelationships(resolvedCalls);
+    return { nodes: [], relationships: callRels };
+  }
+}
+const TYPE_TO_KIND = {
+  Function: "function",
+  Class: "class",
+  File: "file",
+  Directory: "directory"
+};
+class SummarizeStage {
+  name() {
+    return "summarize";
+  }
+  process(node) {
+    if (!node.properties?.summary) {
+      const summary = this.summarizeNode(node);
+      if (summary) {
+        node.properties = { ...node.properties, summary };
+      }
+    }
+    return { nodes: [node], relationships: [] };
+  }
+  flush() {
+    return { nodes: [], relationships: [] };
+  }
+  summarizeNode(node) {
+    const kind = TYPE_TO_KIND[node.type];
+    if (!kind) {
+      return `${node.type} ${node.name}`;
+    }
+    const props = node.properties ?? {};
+    return summarizeFromMetadata({
+      name: node.name,
+      kind,
+      signature: props.signature,
+      language: props.language,
+      lineCount: typeof props.start_line === "number" && typeof props.end_line === "number" ? props.end_line - props.start_line + 1 : void 0,
+      receiverType: props.receiver_type,
+      fileName: kind === "file" ? props.path ?? node.name : void 0,
+      childNames: props.childNames,
+      docs: props.docs
+    });
+  }
+}
+const DEFAULT_DRAIN_THRESHOLD = 500;
+class StoreStage {
+  bufferedNodes = [];
+  bufferedRelationships = [];
+  totalNodes = 0;
+  totalRelationships = 0;
+  drainThreshold;
+  constructor(drainThreshold = DEFAULT_DRAIN_THRESHOLD) {
+    this.drainThreshold = drainThreshold;
+  }
+  name() {
+    return "store";
+  }
+  process(node) {
+    this.bufferedNodes.push(node);
+    this.totalNodes++;
+    return { nodes: [], relationships: [] };
+  }
+  /**
+   * Feed relationships from upstream stage mutations.
+   * Call this from the event loop when processing StageEvent 'end' mutations.
+   */
+  addRelationships(rels) {
+    for (let i = 0; i < rels.length; i++) {
+      this.bufferedRelationships.push(rels[i]);
+      this.totalRelationships++;
+    }
+  }
+  /** True when the node buffer has reached the drain threshold. */
+  needsDrain() {
+    return this.bufferedNodes.length >= this.drainThreshold;
+  }
+  /**
+   * Return and clear buffered nodes. The caller should persist these
+   * to the store (importBatch + flush). Called periodically from the
+   * event loop, not just at the end.
+   */
+  drainNodes() {
+    const nodes = this.bufferedNodes;
+    this.bufferedNodes = [];
+    return nodes;
+  }
+  /**
+   * Return and clear buffered relationships. Called once at the end
+   * after all nodes have been persisted.
+   */
+  drainRelationships() {
+    const rels = this.bufferedRelationships;
+    this.bufferedRelationships = [];
+    return rels;
+  }
+  flush() {
+    return {
+      nodes: this.bufferedNodes,
+      relationships: this.bufferedRelationships
+    };
+  }
+  /** Cumulative counts (including already-drained items). */
+  stats() {
+    return {
+      nodes: this.totalNodes,
+      relationships: this.totalRelationships
+    };
+  }
+}
+class PipelineDebugLog {
+  entries = [];
+  startTime = 0;
+  maxEntries;
+  _enabled;
+  constructor(opts = {}) {
+    this.maxEntries = opts.maxEntries ?? 2e3;
+    this._enabled = opts.enabled ?? true;
+  }
+  get enabled() {
+    return this._enabled;
+  }
+  start() {
+    this.entries = [];
+    this.startTime = performance.now();
+    this.log("pipeline", "started");
+  }
+  log(label, detail) {
+    if (!this._enabled) return;
+    const now = performance.now();
+    const entry = {
+      ts: now,
+      elapsed: now - this.startTime,
+      label,
+      detail
+    };
+    this.entries.push(entry);
+    if (this.entries.length > this.maxEntries) {
+      this.entries.shift();
+    }
+  }
+  logEvent(event) {
+    if (!this._enabled) return;
+    if ("action" in event) {
+      const mutInfo = event.mutation ? ` nodes=${event.mutation.nodes.length} rels=${event.mutation.relationships.length}` : "";
+      this.log(
+        `stage:${event.stage}`,
+        `${event.action} ${event.node}${mutInfo}`
+      );
+    } else if ("kind" in event) {
+      switch (event.kind) {
+        case "pipeline_done":
+          this.log(
+            "pipeline",
+            `done nodes=${event.totalNodes} rels=${event.totalRelationships}`
+          );
+          break;
+        case "pipeline_error":
+          this.log("pipeline", `error: ${event.error}`);
+          break;
+        case "item_error":
+          this.log(
+            `stage:${event.stage}`,
+            `item_error ${event.node}: ${event.error}`
+          );
+          break;
+        case "flush_start":
+          this.log(`stage:${event.stage}`, "flush_start");
+          break;
+        case "flush_end": {
+          const mutInfo = event.mutation ? ` nodes=${event.mutation.nodes.length} rels=${event.mutation.relationships.length}` : "";
+          this.log(`stage:${event.stage}`, `flush_end${mutInfo}`);
+          break;
+        }
+      }
+    }
+  }
+  /** Return all entries (most recent last). */
+  getEntries() {
+    return this.entries;
+  }
+  /** Summarize stage durations and counts. */
+  summary() {
+    const stages = {};
+    for (const entry of this.entries) {
+      if (!entry.label.startsWith("stage:")) continue;
+      const stage = entry.label;
+      if (!stages[stage]) {
+        stages[stage] = { count: 0, totalMs: 0, lastStart: 0 };
+      }
+      if (entry.detail?.startsWith("start ")) {
+        stages[stage].lastStart = entry.ts;
+      } else if (entry.detail?.startsWith("end ")) {
+        if (stages[stage].lastStart > 0) {
+          stages[stage].totalMs += entry.ts - stages[stage].lastStart;
+          stages[stage].count++;
+          stages[stage].lastStart = 0;
+        }
+      }
+    }
+    const result = {};
+    for (const [k, v] of Object.entries(stages)) {
+      result[k] = { count: v.count, totalMs: Math.round(v.totalMs * 100) / 100 };
+    }
+    return result;
+  }
+  /** Dump to console in a readable format. */
+  dump() {
+    console.group("[PipelineDebug] Event log");
+    for (const e of this.entries) {
+      console.log(
+        `%c+${e.elapsed.toFixed(1)}ms%c ${e.label} %c${e.detail ?? ""}`,
+        "color: gray",
+        "color: white; font-weight: bold",
+        "color: cyan"
+      );
+    }
+    console.groupEnd();
+    const s = this.summary();
+    if (Object.keys(s).length > 0) {
+      console.group("[PipelineDebug] Stage summary");
+      for (const [stage, info] of Object.entries(s)) {
+        console.log(
+          `${stage}: ${info.count} items in ${info.totalMs.toFixed(1)}ms (avg ${(info.totalMs / Math.max(info.count, 1)).toFixed(1)}ms)`
+        );
+      }
+      console.groupEnd();
+    }
+  }
+}
 exports.DEFAULT_SUMMARIZER_CONFIG = DEFAULT_SUMMARIZER_CONFIG;
+exports.EMPTY_MUTATION = EMPTY_MUTATION;
+exports.ExtractStage = ExtractStage;
+exports.FileCacheStage = FileCacheStage;
 exports.MemoryStore = MemoryStore;
+exports.PipelineDebugLog = PipelineDebugLog;
+exports.ResolveStage = ResolveStage;
+exports.StoreStage = StoreStage;
+exports.SummarizeStage = SummarizeStage;
 exports.TemplateSummarizer = TemplateSummarizer;
 exports.addToRegistry = addToRegistry;
 exports.analyzeGoImports = analyzeGoImports;
@@ -3445,6 +3974,7 @@ exports.collectPipeline = collectPipeline;
 exports.countSymbols = countSymbols;
 exports.detectLanguage = detectLanguage;
 exports.ensureDirChain = ensureDirChain;
+exports.executeScanning = execute$3;
 exports.extractGeneric = extractGeneric;
 exports.extractGo = extractGo;
 exports.extractKeywords = extractKeywords;
@@ -3471,6 +4001,7 @@ exports.processSymbol = processSymbol;
 exports.resetDirIndexCache = resetDirIndexCache;
 exports.resolveCalls = resolveCalls;
 exports.resolvedCallsToRelationships = resolvedCallsToRelationships;
+exports.runNodePipeline = runNodePipeline;
 exports.runPipeline = runPipeline;
 exports.splitIdentifier = splitIdentifier;
 exports.summarizeClass = summarizeClass;