@neat.is/core 0.2.5

package/dist/server.d.cts

@@ -0,0 +1,2 @@
+
+export {  }

package/dist/server.d.ts

@@ -0,0 +1,2 @@
+
+export {  }

package/dist/server.js

@@ -0,0 +1,97 @@
+import {
+  buildSearchIndex
+} from "./chunk-XOOCA5T7.js";
+import {
+  buildApi
+} from "./chunk-T2U4U256.js";
+import {
+  DEFAULT_PROJECT,
+  Projects,
+  extractFromDirectory,
+  getGraph,
+  loadGraphFromDisk,
+  makeSpanHandler,
+  parseExtraProjects,
+  pathsForProject,
+  startPersistLoop,
+  startStalenessLoop
+} from "./chunk-6SFEITLJ.js";
+import {
+  buildOtelReceiver,
+  startOtelGrpcReceiver
+} from "./chunk-I5IMCXRO.js";
+
+// src/server.ts
+import path from "path";
+async function bootProject(registry, name, scanPath, baseDir) {
+  const paths = pathsForProject(name, baseDir);
+  const graph = getGraph(name);
+  await loadGraphFromDisk(graph, paths.snapshotPath);
+  if (scanPath) {
+    const r = await extractFromDirectory(graph, scanPath);
+    console.log(
+      `[${name}] extract: ${r.nodesAdded} new nodes, ${r.edgesAdded} new edges (graph total ${graph.order}/${graph.size})`
+    );
+  } else {
+    console.log(`[${name}] loaded ${graph.order} nodes / ${graph.size} edges from snapshot`);
+  }
+  startPersistLoop(graph, paths.snapshotPath);
+  startStalenessLoop(graph, { staleEventsPath: paths.staleEventsPath });
+  const searchIndex = await buildSearchIndex(graph, {
+    cachePath: paths.embeddingsCachePath
+  }).catch((err) => {
+    console.warn(
+      `[${name}] semantic_search: index build failed (${err.message}); falling back to inline substring`
+    );
+    return void 0;
+  });
+  if (searchIndex) {
+    console.log(`[${name}] semantic_search: ${searchIndex.provider} provider`);
+  }
+  registry.set(name, {
+    scanPath,
+    paths,
+    searchIndex
+  });
+}
+async function main() {
+  const baseDirEnv = process.env.NEAT_OUT_DIR;
+  const legacyOutPath = process.env.NEAT_OUT_PATH;
+  const baseDir = baseDirEnv ? path.resolve(baseDirEnv) : legacyOutPath ? path.resolve(path.dirname(legacyOutPath)) : path.resolve("./neat-out");
+  const defaultScanPath = path.resolve(process.env.NEAT_SCAN_PATH ?? "./demo");
+  const registry = new Projects();
+  await bootProject(registry, DEFAULT_PROJECT, defaultScanPath, baseDir);
+  for (const name of parseExtraProjects(process.env.NEAT_PROJECTS)) {
+    const envKey = `NEAT_PROJECT_SCAN_PATH_${name.toUpperCase().replace(/[^A-Z0-9]/g, "_")}`;
+    const projectScan = process.env[envKey];
+    await bootProject(registry, name, projectScan ? path.resolve(projectScan) : void 0, baseDir);
+  }
+  const host = process.env.HOST ?? "0.0.0.0";
+  const port = Number(process.env.PORT ?? 8080);
+  const otelPort = Number(process.env.OTEL_PORT ?? 4318);
+  const app = await buildApi({ projects: registry });
+  await app.listen({ port, host });
+  console.log(`neat-core listening on http://${host}:${port}`);
+  console.log(`  base dir:      ${baseDir}`);
+  console.log(`  projects:      ${registry.list().join(", ")}`);
+  const defaultCtx = registry.get(DEFAULT_PROJECT);
+  if (defaultCtx) {
+    const onSpan = makeSpanHandler({
+      graph: defaultCtx.graph,
+      errorsPath: defaultCtx.paths.errorsPath
+    });
+    const otelApp = await buildOtelReceiver({ onSpan });
+    await otelApp.listen({ port: otelPort, host });
+    console.log(`neat-core OTLP receiver on http://${host}:${otelPort}/v1/traces`);
+    if (process.env.NEAT_OTLP_GRPC === "true") {
+      const grpcPort = Number(process.env.NEAT_OTLP_GRPC_PORT ?? 4317);
+      const grpcReceiver = await startOtelGrpcReceiver({ onSpan, host, port: grpcPort });
+      console.log(`neat-core OTLP/gRPC receiver on ${grpcReceiver.address}`);
+    }
+  }
+}
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});
+//# sourceMappingURL=server.js.map

package/dist/server.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/server.ts"],"sourcesContent":["import path from 'node:path'\nimport { DEFAULT_PROJECT, getGraph } from './graph.js'\nimport { buildApi } from './api.js'\nimport { extractFromDirectory } from './extract.js'\nimport { loadGraphFromDisk, startPersistLoop } from './persist.js'\nimport { buildOtelReceiver } from './otel.js'\nimport { startOtelGrpcReceiver } from './otel-grpc.js'\nimport { makeSpanHandler, startStalenessLoop } from './ingest.js'\nimport { buildSearchIndex } from './search.js'\nimport { Projects, parseExtraProjects, pathsForProject } from './projects.js'\n\nasync function bootProject(\n  registry: Projects,\n  name: string,\n  scanPath: string | undefined,\n  baseDir: string,\n): Promise<void> {\n  const paths = pathsForProject(name, baseDir)\n  const graph = getGraph(name)\n  await loadGraphFromDisk(graph, paths.snapshotPath)\n\n  if (scanPath) {\n    const r = await extractFromDirectory(graph, scanPath)\n    console.log(\n      `[${name}] extract: ${r.nodesAdded} new nodes, ${r.edgesAdded} new edges (graph total ${graph.order}/${graph.size})`,\n    )\n  } else {\n    console.log(`[${name}] loaded ${graph.order} nodes / ${graph.size} edges from snapshot`)\n  }\n\n  startPersistLoop(graph, paths.snapshotPath)\n  startStalenessLoop(graph, { staleEventsPath: paths.staleEventsPath })\n\n  const searchIndex = await buildSearchIndex(graph, {\n    cachePath: paths.embeddingsCachePath,\n  }).catch((err) => {\n    console.warn(\n      `[${name}] semantic_search: index build failed (${(err as Error).message}); falling back to inline substring`,\n    )\n    return undefined\n  })\n  if (searchIndex) {\n    console.log(`[${name}] semantic_search: ${searchIndex.provider} provider`)\n  }\n\n  registry.set(name, {\n    scanPath,\n    paths,\n    searchIndex,\n  })\n}\n\nasync function main(): Promise<void> {\n  const baseDirEnv = process.env.NEAT_OUT_DIR\n  const legacyOutPath = process.env.NEAT_OUT_PATH\n  const baseDir = baseDirEnv\n    ? path.resolve(baseDirEnv)\n    : legacyOutPath\n      ? path.resolve(path.dirname(legacyOutPath))\n      : path.resolve('./neat-out')\n\n  const defaultScanPath = path.resolve(process.env.NEAT_SCAN_PATH ?? './demo')\n  const registry = new Projects()\n\n  // Default project always exists. NEAT_SCAN_PATH still wires it to a scan\n  // root so existing single-project users see no behaviour change.\n  await bootProject(registry, DEFAULT_PROJECT, defaultScanPath, baseDir)\n\n  // Extra projects come from NEAT_PROJECTS=a,b,c. Their snapshots load from\n  // <baseDir>/<name>.json; they have no scan path by default (callers can\n  // POST /projects/<name>/graph/scan after wiring NEAT_PROJECT_SCAN_PATH_<NAME>).\n  for (const name of parseExtraProjects(process.env.NEAT_PROJECTS)) {\n    const envKey = `NEAT_PROJECT_SCAN_PATH_${name.toUpperCase().replace(/[^A-Z0-9]/g, '_')}`\n    const projectScan = process.env[envKey]\n    await bootProject(registry, name, projectScan ? path.resolve(projectScan) : undefined, baseDir)\n  }\n\n  const host = process.env.HOST ?? '0.0.0.0'\n  const port = Number(process.env.PORT ?? 8080)\n  const otelPort = Number(process.env.OTEL_PORT ?? 4318)\n\n  const app = await buildApi({ projects: registry })\n  await app.listen({ port, host })\n  console.log(`neat-core listening on http://${host}:${port}`)\n  console.log(`  base dir:      ${baseDir}`)\n  console.log(`  projects:      ${registry.list().join(', ')}`)\n\n  // OTel ingest stays single-project for now: spans always land in the\n  // default project's graph + errors log. Multi-project routing for spans\n  // is a future concern (would need a header / resource attr).\n  const defaultCtx = registry.get(DEFAULT_PROJECT)\n  if (defaultCtx) {\n    const onSpan = makeSpanHandler({\n      graph: defaultCtx.graph,\n      errorsPath: defaultCtx.paths.errorsPath,\n    })\n    const otelApp = await buildOtelReceiver({ onSpan })\n    await otelApp.listen({ port: otelPort, host })\n    console.log(`neat-core OTLP receiver on http://${host}:${otelPort}/v1/traces`)\n\n    if (process.env.NEAT_OTLP_GRPC === 'true') {\n      const grpcPort = Number(process.env.NEAT_OTLP_GRPC_PORT ?? 4317)\n      const grpcReceiver = await startOtelGrpcReceiver({ onSpan, host, port: grpcPort })\n      console.log(`neat-core OTLP/gRPC receiver on ${grpcReceiver.address}`)\n    }\n  }\n}\n\nmain().catch((err) => {\n  console.error(err)\n  process.exit(1)\n})\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,OAAO,UAAU;AAWjB,eAAe,YACb,UACA,MACA,UACA,SACe;AACf,QAAM,QAAQ,gBAAgB,MAAM,OAAO;AAC3C,QAAM,QAAQ,SAAS,IAAI;AAC3B,QAAM,kBAAkB,OAAO,MAAM,YAAY;AAEjD,MAAI,UAAU;AACZ,UAAM,IAAI,MAAM,qBAAqB,OAAO,QAAQ;AACpD,YAAQ;AAAA,MACN,IAAI,IAAI,cAAc,EAAE,UAAU,eAAe,EAAE,UAAU,2BAA2B,MAAM,KAAK,IAAI,MAAM,IAAI;AAAA,IACnH;AAAA,EACF,OAAO;AACL,YAAQ,IAAI,IAAI,IAAI,YAAY,MAAM,KAAK,YAAY,MAAM,IAAI,sBAAsB;AAAA,EACzF;AAEA,mBAAiB,OAAO,MAAM,YAAY;AAC1C,qBAAmB,OAAO,EAAE,iBAAiB,MAAM,gBAAgB,CAAC;AAEpE,QAAM,cAAc,MAAM,iBAAiB,OAAO;AAAA,IAChD,WAAW,MAAM;AAAA,EACnB,CAAC,EAAE,MAAM,CAAC,QAAQ;AAChB,YAAQ;AAAA,MACN,IAAI,IAAI,0CAA2C,IAAc,OAAO;AAAA,IAC1E;AACA,WAAO;AAAA,EACT,CAAC;AACD,MAAI,aAAa;AACf,YAAQ,IAAI,IAAI,IAAI,sBAAsB,YAAY,QAAQ,WAAW;AAAA,EAC3E;AAEA,WAAS,IAAI,MAAM;AAAA,IACjB;AAAA,IACA;AAAA,IACA;AAAA,EACF,CAAC;AACH;AAEA,eAAe,OAAsB;AACnC,QAAM,aAAa,QAAQ,IAAI;AAC/B,QAAM,gBAAgB,QAAQ,IAAI;AAClC,QAAM,UAAU,aACZ,KAAK,QAAQ,UAAU,IACvB,gBACE,KAAK,QAAQ,KAAK,QAAQ,aAAa,CAAC,IACxC,KAAK,QAAQ,YAAY;AAE/B,QAAM,kBAAkB,KAAK,QAAQ,QAAQ,IAAI,kBAAkB,QAAQ;AAC3E,QAAM,WAAW,IAAI,SAAS;AAI9B,QAAM,YAAY,UAAU,iBAAiB,iBAAiB,OAAO;AAKrE,aAAW,QAAQ,mBAAmB,QAAQ,IAAI,aAAa,GAAG;AAChE,UAAM,SAAS,0BAA0B,KAAK,YAAY,EAAE,QAAQ,cAAc,GAAG,CAAC;AACtF,UAAM,cAAc,QAAQ,IAAI,MAAM;AACtC,UAAM,YAAY,UAAU,MAAM,cAAc,KAAK,QAAQ,WAAW,IAAI,QAAW,OAAO;AAAA,EAChG;AAEA,QAAM,OAAO,QAAQ,IAAI,QAAQ;AACjC,QAAM,OAAO,OAAO,QAAQ,IAAI,QAAQ,IAAI;AAC5C,QAAM,WAAW,OAAO,QAAQ,IAAI,aAAa,IAAI;AAErD,QAAM,MAAM,MAAM,SAAS,EAAE,UAAU,SAAS,CAAC;AACjD,QAAM,IAAI,OAAO,EAAE,MAAM,KAAK,CAAC;AAC/B,UAAQ,IAAI,iCAAiC,IAAI,IAAI,IAAI,EAAE;AAC3D,UAAQ,IAAI,oBAAoB,OAAO,EAAE;AACzC,UAAQ,IAAI,oBAAoB,SAAS,KAAK,EAAE,KAAK,IAAI,CAAC,EAAE;AAK5D,QAAM,aAAa,SAAS,IAAI,eAAe;AAC/C,MAAI,YAAY;AACd,UAAM,SAAS,gBAAgB;AAAA,MAC7B,OAAO,WAAW;AAAA,MAClB,YAAY,WAAW,MAAM;AAAA,IAC/B,CAAC;AACD,UAAM,UAAU,MAAM,kBAAkB,EAAE,OAAO,CAAC;AAClD,UAAM,QAAQ,OAAO,EAAE,MAAM,UAAU,KAAK,CAAC;AAC7C,YAAQ,IAAI,qCAAqC,IAAI,IAAI,QAAQ,YAAY;AAE7E,QAAI,QAAQ,IAAI,mBAAmB,QAAQ;AACzC,YAAM,WAAW,OAAO,QAAQ,IAAI,uBAAuB,IAAI;AAC/D,YAAM,eAAe,MAAM,sBAAsB,EAAE,QAAQ,MAAM,MAAM,SAAS,CAAC;AACjF,cAAQ,IAAI,mCAAmC,aAAa,OAAO,EAAE;AAAA,IACvE;AAAA,EACF;AACF;AAEA,KAAK,EAAE,MAAM,CAAC,QAAQ;AACpB,UAAQ,MAAM,GAAG;AACjB,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":[]}

package/package.json

@@ -0,0 +1,77 @@
+{
+  "name": "@neat.is/core",
+  "version": "0.2.5",
+  "description": "NEAT graph engine: tree-sitter extraction, OTel ingest, REST API",
+  "license": "BUSL-1.1",
+  "homepage": "https://neat.is",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/NEAT-Technologies/Neat.git",
+    "directory": "packages/core"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
+    }
+  },
+  "bin": {
+    "neat": "./dist/cli.cjs",
+    "neatd": "./dist/neatd.cjs"
+  },
+  "files": [
+    "dist",
+    "compat.json",
+    "proto"
+  ],
+  "scripts": {
+    "build": "tsup",
+    "prepublishOnly": "npm run build",
+    "dev": "tsx watch src/server.ts",
+    "start": "node dist/server.cjs",
+    "test": "vitest run",
+    "lint": "eslint src",
+    "clean": "rm -rf dist .turbo"
+  },
+  "dependencies": {
+    "@fastify/cors": "^10.0.0",
+    "@grpc/grpc-js": "^1.14.3",
+    "@grpc/proto-loader": "^0.8.0",
+    "@neat.is/types": "^0.2.5",
+    "chokidar": "^4.0.3",
+    "fastify": "^5.0.0",
+    "graphology": "^0.25.4",
+    "graphology-shortest-path": "^2.1.0",
+    "graphology-traversal": "^0.3.1",
+    "ignore": "^7.0.0",
+    "minimatch": "^10.0.0",
+    "protobufjs": "^7.5.6",
+    "semver": "^7.6.0",
+    "smol-toml": "^1.3.0",
+    "tree-sitter": "^0.21.1",
+    "tree-sitter-javascript": "^0.21.4",
+    "tree-sitter-python": "^0.21.0",
+    "tree-sitter-typescript": "^0.21.2",
+    "yaml": "^2.5.0",
+    "zod": "^3.23.8"
+  },
+  "devDependencies": {
+    "@types/node": "^20.14.0",
+    "@types/semver": "^7.5.8",
+    "tsup": "^8.3.0",
+    "tsx": "^4.19.0",
+    "typescript": "^5.6.0",
+    "vitest": "^2.1.0"
+  },
+  "optionalDependencies": {
+    "@xenova/transformers": "^2.17.2"
+  }
+}

package/proto/opentelemetry/proto/collector/trace/v1/trace_service.proto

@@ -0,0 +1,31 @@
+// Copyright 2019, OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+//
+// Bundled from https://github.com/open-telemetry/opentelemetry-proto.
+syntax = "proto3";
+
+package opentelemetry.proto.collector.trace.v1;
+
+import "opentelemetry/proto/trace/v1/trace.proto";
+
+option java_multiple_files = true;
+option java_package = "io.opentelemetry.proto.collector.trace.v1";
+option java_outer_classname = "TraceServiceProto";
+option go_package = "go.opentelemetry.io/proto/otlp/collector/trace/v1";
+
+service TraceService {
+  rpc Export(ExportTraceServiceRequest) returns (ExportTraceServiceResponse) {}
+}
+
+message ExportTraceServiceRequest {
+  repeated opentelemetry.proto.trace.v1.ResourceSpans resource_spans = 1;
+}
+
+message ExportTraceServiceResponse {
+  ExportTracePartialSuccess partial_success = 1;
+}
+
+message ExportTracePartialSuccess {
+  int64 rejected_spans = 1;
+  string error_message = 2;
+}

package/proto/opentelemetry/proto/common/v1/common.proto

@@ -0,0 +1,46 @@
+// Copyright 2019, OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+//
+// Bundled from https://github.com/open-telemetry/opentelemetry-proto.
+// We carry this in-tree so @grpc/proto-loader can load OTLP at runtime
+// without an extra package dependency.
+syntax = "proto3";
+
+package opentelemetry.proto.common.v1;
+
+option java_multiple_files = true;
+option java_package = "io.opentelemetry.proto.common.v1";
+option java_outer_classname = "CommonProto";
+option go_package = "go.opentelemetry.io/proto/otlp/common/v1";
+
+message AnyValue {
+  oneof value {
+    string string_value = 1;
+    bool bool_value = 2;
+    int64 int_value = 3;
+    double double_value = 4;
+    ArrayValue array_value = 5;
+    KeyValueList kvlist_value = 6;
+    bytes bytes_value = 7;
+  }
+}
+
+message ArrayValue {
+  repeated AnyValue values = 1;
+}
+
+message KeyValueList {
+  repeated KeyValue values = 1;
+}
+
+message KeyValue {
+  string key = 1;
+  AnyValue value = 2;
+}
+
+message InstrumentationScope {
+  string name = 1;
+  string version = 2;
+  repeated KeyValue attributes = 3;
+  uint32 dropped_attributes_count = 4;
+}

package/proto/opentelemetry/proto/resource/v1/resource.proto

@@ -0,0 +1,19 @@
+// Copyright 2019, OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+//
+// Bundled from https://github.com/open-telemetry/opentelemetry-proto.
+syntax = "proto3";
+
+package opentelemetry.proto.resource.v1;
+
+import "opentelemetry/proto/common/v1/common.proto";
+
+option java_multiple_files = true;
+option java_package = "io.opentelemetry.proto.resource.v1";
+option java_outer_classname = "ResourceProto";
+option go_package = "go.opentelemetry.io/proto/otlp/resource/v1";
+
+message Resource {
+  repeated opentelemetry.proto.common.v1.KeyValue attributes = 1;
+  uint32 dropped_attributes_count = 2;
+}

package/proto/opentelemetry/proto/trace/v1/trace.proto

@@ -0,0 +1,93 @@
+// Copyright 2019, OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+//
+// Bundled from https://github.com/open-telemetry/opentelemetry-proto.
+syntax = "proto3";
+
+package opentelemetry.proto.trace.v1;
+
+import "opentelemetry/proto/common/v1/common.proto";
+import "opentelemetry/proto/resource/v1/resource.proto";
+
+option java_multiple_files = true;
+option java_package = "io.opentelemetry.proto.trace.v1";
+option java_outer_classname = "TraceProto";
+option go_package = "go.opentelemetry.io/proto/otlp/trace/v1";
+
+message TracesData {
+  repeated ResourceSpans resource_spans = 1;
+}
+
+message ResourceSpans {
+  reserved 1000;
+  opentelemetry.proto.resource.v1.Resource resource = 1;
+  repeated ScopeSpans scope_spans = 2;
+  string schema_url = 3;
+}
+
+message ScopeSpans {
+  opentelemetry.proto.common.v1.InstrumentationScope scope = 1;
+  repeated Span spans = 2;
+  string schema_url = 3;
+}
+
+message Span {
+  reserved 3;
+  bytes trace_id = 1;
+  bytes span_id = 2;
+  string trace_state = 3;
+  bytes parent_span_id = 4;
+  string name = 5;
+
+  enum SpanKind {
+    SPAN_KIND_UNSPECIFIED = 0;
+    SPAN_KIND_INTERNAL = 1;
+    SPAN_KIND_SERVER = 2;
+    SPAN_KIND_CLIENT = 3;
+    SPAN_KIND_PRODUCER = 4;
+    SPAN_KIND_CONSUMER = 5;
+  }
+
+  SpanKind kind = 6;
+  fixed64 start_time_unix_nano = 7;
+  fixed64 end_time_unix_nano = 8;
+  repeated opentelemetry.proto.common.v1.KeyValue attributes = 9;
+  uint32 dropped_attributes_count = 10;
+
+  message Event {
+    fixed64 time_unix_nano = 1;
+    string name = 2;
+    repeated opentelemetry.proto.common.v1.KeyValue attributes = 3;
+    uint32 dropped_attributes_count = 4;
+  }
+
+  repeated Event events = 11;
+  uint32 dropped_events_count = 12;
+
+  message Link {
+    bytes trace_id = 1;
+    bytes span_id = 2;
+    string trace_state = 3;
+    repeated opentelemetry.proto.common.v1.KeyValue attributes = 4;
+    uint32 dropped_attributes_count = 5;
+    uint32 flags = 6;
+  }
+
+  repeated Link links = 13;
+  uint32 dropped_links_count = 14;
+  Status status = 15;
+  uint32 flags = 16;
+}
+
+message Status {
+  reserved 1;
+  string message = 2;
+
+  enum StatusCode {
+    STATUS_CODE_UNSET = 0;
+    STATUS_CODE_OK = 1;
+    STATUS_CODE_ERROR = 2;
+  }
+
+  StatusCode code = 3;
+}