@gracker/smartperfetto 1.0.14 → 1.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +16 -8
- package/dist/agent/core/orchestratorTypes.d.ts +10 -0
- package/dist/agent/core/orchestratorTypes.d.ts.map +1 -1
- package/dist/agent/core/orchestratorTypes.js.map +1 -1
- package/dist/agentOpenAI/openAiRuntime.d.ts.map +1 -1
- package/dist/agentOpenAI/openAiRuntime.js +11 -1
- package/dist/agentOpenAI/openAiRuntime.js.map +1 -1
- package/dist/agentOpenAI/openAiToolAdapter.d.ts.map +1 -1
- package/dist/agentOpenAI/openAiToolAdapter.js +27 -2
- package/dist/agentOpenAI/openAiToolAdapter.js.map +1 -1
- package/dist/agentv3/claudeMcpServer.d.ts +15 -1
- package/dist/agentv3/claudeMcpServer.d.ts.map +1 -1
- package/dist/agentv3/claudeMcpServer.js +311 -5
- package/dist/agentv3/claudeMcpServer.js.map +1 -1
- package/dist/agentv3/claudeRuntime.d.ts.map +1 -1
- package/dist/agentv3/claudeRuntime.js +17 -0
- package/dist/agentv3/claudeRuntime.js.map +1 -1
- package/dist/agentv3/claudeSystemPrompt.d.ts.map +1 -1
- package/dist/agentv3/claudeSystemPrompt.js +9 -0
- package/dist/agentv3/claudeSystemPrompt.js.map +1 -1
- package/dist/agentv3/mcpToolRegistry.d.ts +14 -2
- package/dist/agentv3/mcpToolRegistry.d.ts.map +1 -1
- package/dist/agentv3/mcpToolRegistry.js +25 -6
- package/dist/agentv3/mcpToolRegistry.js.map +1 -1
- package/dist/agentv3/sessionStateSnapshot.d.ts +18 -0
- package/dist/agentv3/sessionStateSnapshot.d.ts.map +1 -1
- package/dist/agentv3/types.d.ts +5 -0
- package/dist/agentv3/types.d.ts.map +1 -1
- package/dist/agentv3/types.js.map +1 -1
- package/dist/cli-user/bin.js +99 -1
- package/dist/cli-user/bin.js.map +1 -1
- package/dist/cli-user/commands/analyze.d.ts +3 -0
- package/dist/cli-user/commands/analyze.d.ts.map +1 -1
- package/dist/cli-user/commands/analyze.js +2 -0
- package/dist/cli-user/commands/analyze.js.map +1 -1
- package/dist/cli-user/commands/codebase.d.ts +29 -0
- package/dist/cli-user/commands/codebase.d.ts.map +1 -0
- package/dist/cli-user/commands/codebase.js +152 -0
- package/dist/cli-user/commands/codebase.js.map +1 -0
- package/dist/cli-user/commands/config.d.ts.map +1 -1
- package/dist/cli-user/commands/config.js +4 -1
- package/dist/cli-user/commands/config.js.map +1 -1
- package/dist/cli-user/services/cliAnalyzeService.d.ts +3 -0
- package/dist/cli-user/services/cliAnalyzeService.d.ts.map +1 -1
- package/dist/cli-user/services/cliAnalyzeService.js +116 -29
- package/dist/cli-user/services/cliAnalyzeService.js.map +1 -1
- package/dist/cli-user/services/turnRunner.d.ts +3 -0
- package/dist/cli-user/services/turnRunner.d.ts.map +1 -1
- package/dist/cli-user/services/turnRunner.js +6 -0
- package/dist/cli-user/services/turnRunner.js.map +1 -1
- package/dist/cli-user/types.d.ts +5 -0
- package/dist/cli-user/types.d.ts.map +1 -1
- package/dist/routes/agentRoutes.d.ts.map +1 -1
- package/dist/routes/agentRoutes.js +4 -0
- package/dist/routes/agentRoutes.js.map +1 -1
- package/dist/routes/ragAdminRoutes.d.ts +13 -24
- package/dist/routes/ragAdminRoutes.d.ts.map +1 -1
- package/dist/routes/ragAdminRoutes.js +266 -11
- package/dist/routes/ragAdminRoutes.js.map +1 -1
- package/dist/scripts/verifyAgentSseScrolling.js +119 -3
- package/dist/scripts/verifyAgentSseScrolling.js.map +1 -1
- package/dist/services/aospKnowledgeIngester.d.ts +7 -0
- package/dist/services/aospKnowledgeIngester.d.ts.map +1 -1
- package/dist/services/aospKnowledgeIngester.js +19 -7
- package/dist/services/aospKnowledgeIngester.js.map +1 -1
- package/dist/services/auth/codebaseScopes.d.ts +6 -0
- package/dist/services/auth/codebaseScopes.d.ts.map +1 -0
- package/dist/services/auth/codebaseScopes.js +23 -0
- package/dist/services/auth/codebaseScopes.js.map +1 -0
- package/dist/services/codebase/codeAwareFeature.d.ts +4 -0
- package/dist/services/codebase/codeAwareFeature.d.ts.map +1 -0
- package/dist/services/codebase/codeAwareFeature.js +18 -0
- package/dist/services/codebase/codeAwareFeature.js.map +1 -0
- package/dist/services/codebase/codeLookupLedger.d.ts +37 -0
- package/dist/services/codebase/codeLookupLedger.d.ts.map +1 -0
- package/dist/services/codebase/codeLookupLedger.js +126 -0
- package/dist/services/codebase/codeLookupLedger.js.map +1 -0
- package/dist/services/codebase/codebaseRegistry.d.ts +76 -0
- package/dist/services/codebase/codebaseRegistry.d.ts.map +1 -0
- package/dist/services/codebase/codebaseRegistry.js +164 -0
- package/dist/services/codebase/codebaseRegistry.js.map +1 -0
- package/dist/services/codebase/defaultCodebaseServices.d.ts +5 -0
- package/dist/services/codebase/defaultCodebaseServices.d.ts.map +1 -0
- package/dist/services/codebase/defaultCodebaseServices.js +21 -0
- package/dist/services/codebase/defaultCodebaseServices.js.map +1 -0
- package/dist/services/codebase/patchProposer.d.ts +44 -0
- package/dist/services/codebase/patchProposer.d.ts.map +1 -0
- package/dist/services/codebase/patchProposer.js +163 -0
- package/dist/services/codebase/patchProposer.js.map +1 -0
- package/dist/services/codebase/pathSecurityGate.d.ts +33 -0
- package/dist/services/codebase/pathSecurityGate.d.ts.map +1 -0
- package/dist/services/codebase/pathSecurityGate.js +214 -0
- package/dist/services/codebase/pathSecurityGate.js.map +1 -0
- package/dist/services/htmlReportGenerator.d.ts +2 -0
- package/dist/services/htmlReportGenerator.d.ts.map +1 -1
- package/dist/services/htmlReportGenerator.js +67 -0
- package/dist/services/htmlReportGenerator.js.map +1 -1
- package/dist/services/persistAgentSession.d.ts.map +1 -1
- package/dist/services/persistAgentSession.js +19 -0
- package/dist/services/persistAgentSession.js.map +1 -1
- package/dist/services/rag/aospSourceIngester.d.ts +26 -0
- package/dist/services/rag/aospSourceIngester.d.ts.map +1 -0
- package/dist/services/rag/aospSourceIngester.js +143 -0
- package/dist/services/rag/aospSourceIngester.js.map +1 -0
- package/dist/services/rag/appSourceIngester.d.ts +37 -0
- package/dist/services/rag/appSourceIngester.d.ts.map +1 -0
- package/dist/services/rag/appSourceIngester.js +165 -0
- package/dist/services/rag/appSourceIngester.js.map +1 -0
- package/dist/services/rag/baseIngester.d.ts +18 -0
- package/dist/services/rag/baseIngester.d.ts.map +1 -0
- package/dist/services/rag/baseIngester.js +197 -0
- package/dist/services/rag/baseIngester.js.map +1 -0
- package/dist/services/rag/kernelSourceIngester.d.ts +35 -0
- package/dist/services/rag/kernelSourceIngester.d.ts.map +1 -0
- package/dist/services/rag/kernelSourceIngester.js +170 -0
- package/dist/services/rag/kernelSourceIngester.js.map +1 -0
- package/dist/services/rag/lookupResponseFilter.d.ts +42 -0
- package/dist/services/rag/lookupResponseFilter.d.ts.map +1 -0
- package/dist/services/rag/lookupResponseFilter.js +192 -0
- package/dist/services/rag/lookupResponseFilter.js.map +1 -0
- package/dist/services/rag/sessionToolResultRegistry.d.ts +25 -0
- package/dist/services/rag/sessionToolResultRegistry.d.ts.map +1 -0
- package/dist/services/rag/sessionToolResultRegistry.js +57 -0
- package/dist/services/rag/sessionToolResultRegistry.js.map +1 -0
- package/dist/services/rag/toolResultProjectionFilter.d.ts +18 -0
- package/dist/services/rag/toolResultProjectionFilter.d.ts.map +1 -0
- package/dist/services/rag/toolResultProjectionFilter.js +33 -0
- package/dist/services/rag/toolResultProjectionFilter.js.map +1 -0
- package/dist/services/ragStore.d.ts +8 -0
- package/dist/services/ragStore.d.ts.map +1 -1
- package/dist/services/ragStore.js +77 -8
- package/dist/services/ragStore.js.map +1 -1
- package/dist/services/rbac.d.ts +1 -1
- package/dist/services/rbac.d.ts.map +1 -1
- package/dist/services/rbac.js +6 -0
- package/dist/services/rbac.js.map +1 -1
- package/dist/services/security/codeAwareOutputRegistry.d.ts +6 -0
- package/dist/services/security/codeAwareOutputRegistry.d.ts.map +1 -0
- package/dist/services/security/codeAwareOutputRegistry.js +55 -0
- package/dist/services/security/codeAwareOutputRegistry.js.map +1 -0
- package/dist/services/security/llmEchoOutputFilter.d.ts +49 -0
- package/dist/services/security/llmEchoOutputFilter.d.ts.map +1 -0
- package/dist/services/security/llmEchoOutputFilter.js +133 -0
- package/dist/services/security/llmEchoOutputFilter.js.map +1 -0
- package/dist/services/security/secretPatterns.d.ts +6 -0
- package/dist/services/security/secretPatterns.d.ts.map +1 -0
- package/dist/services/security/secretPatterns.js +23 -0
- package/dist/services/security/secretPatterns.js.map +1 -0
- package/dist/services/symbol/breakpadSymParser.d.ts +19 -0
- package/dist/services/symbol/breakpadSymParser.d.ts.map +1 -0
- package/dist/services/symbol/breakpadSymParser.js +40 -0
- package/dist/services/symbol/breakpadSymParser.js.map +1 -0
- package/dist/services/symbol/kallsymsParser.d.ts +12 -0
- package/dist/services/symbol/kallsymsParser.d.ts.map +1 -0
- package/dist/services/symbol/kallsymsParser.js +41 -0
- package/dist/services/symbol/kallsymsParser.js.map +1 -0
- package/dist/services/symbol/r8MappingParser.d.ts +40 -0
- package/dist/services/symbol/r8MappingParser.d.ts.map +1 -0
- package/dist/services/symbol/r8MappingParser.js +99 -0
- package/dist/services/symbol/r8MappingParser.js.map +1 -0
- package/dist/services/symbol/symbolResolver.d.ts +59 -0
- package/dist/services/symbol/symbolResolver.d.ts.map +1 -0
- package/dist/services/symbol/symbolResolver.js +147 -0
- package/dist/services/symbol/symbolResolver.js.map +1 -0
- package/dist/services/symbol/traceSymbolContext.d.ts +14 -0
- package/dist/services/symbol/traceSymbolContext.d.ts.map +1 -0
- package/dist/services/symbol/traceSymbolContext.js +45 -0
- package/dist/services/symbol/traceSymbolContext.js.map +1 -0
- package/dist/types/sparkContracts.d.ts +31 -3
- package/dist/types/sparkContracts.d.ts.map +1 -1
- package/dist/types/sparkContracts.js.map +1 -1
- package/package.json +8 -1
- package/skills/composite/code_pinpoint.skill.yaml +111 -0
- package/strategies/code-aware.template.md +37 -0
- package/strategies/prompt-methodology.template.md +2 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"persistAgentSession.js","sourceRoot":"","sources":["../../src/services/persistAgentSession.ts"],"names":[],"mappings":";AAAA,6CAA6C;AAC7C,0CAA0C;AAC1C,+DAA+D;;
|
|
1
|
+
{"version":3,"file":"persistAgentSession.js","sourceRoot":"","sources":["../../src/services/persistAgentSession.ts"],"names":[],"mappings":";AAAA,6CAA6C;AAC7C,0CAA0C;AAC1C,+DAA+D;;AA4D/D,4CAgJC;AApLD,2EAAwE;AACxE,oFAAgF;AAChF,gFAAgF;AAChF,kEAA+D;AAE/D,SAAS,qBAAqB,CAAC,WAAiC;IAC9D,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC/D,MAAM,QAAQ,GAAG,IAAA,oDAA0B,GAAE,CAAC;IAC9C,OAAO,WAAW;SACf,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;SAC3B,MAAM,CAAC,OAAO,CAAC;SACf,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACX,UAAU,EAAE,GAAI,CAAC,UAAU;QAC3B,eAAe,EAAE,GAAI,CAAC,eAAe;QACrC,WAAW,EAAE,GAAI,CAAC,OAAO,CAAC,WAAW;KACtC,CAAC,CAAC,CAAC;AACR,CAAC;AAoBD,SAAgB,gBAAgB,CAAC,KAA4B;IAC3D,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC;IACrE,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY,IAAI,kBAAkB,CAAC;IAC9D,MAAM,kBAAkB,GAAG,qDAAyB,CAAC,WAAW,EAAE,CAAC;IAEnE,IAAI,CAAC;QACH,MAAM,cAAc,GAAG,8CAAqB,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAEpE,MAAM,QAAQ,GAAG,OAAO,OAAO,CAAC,YAAY,CAAC,YAAY,KAAK,UAAU;YACtE,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,EAAE;gBACrD,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;gBAC1C,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;gBAC1C,uBAAuB,EAAE,OAAO,CAAC,uBAAuB;gBACxD,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,EAAE;gBAClD,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,EAAE;gBACxC,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,EAAE;gBAClD,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,EAAE;gBAC1C,cAAc,EAAE,OAAO,CAAC,cAAc,IAAI,EAAE;gBAC5C,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,EAAE;gBAC1C,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,EAAE;gBACnC,sBAAsB,EAAE,OAAO,CAAC,UAAU;gBAC1C,gCAAgC,EAAE,OAAO,CAAC,oBAAoB;gBAC9D,aAAa,EAAG,OAAqC,CAAC,aAAoB;gBAC1E,WAAW,EAAG,OAAoC,CAAC,WAAW;gBAC9D,gBAAgB,EAAE,qBAAqB,CAAE,OAAoC,CAAC,WAAW,CAAC;gBAC1F,iBAAiB,EAAE,mCAAgB,CAAC,OAAO,CAAC,SAAS,EAAE,KAAM,EAAE,CAAC,CAAC,CAAC,iBAAiB,EAAE;gBACrF,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,CAAC;gBACtC,mBAAmB,EAAE,OAAO,CAAC,mBAAmB,IAAI,CAAC;aACtD,CAAC;YACJ,CAAC,CAAC,IAAI,CAAC;QAET,kEAAkE;QAClE,sEAAsE;QACtE,wCAAwC;QACxC,IAAI,QAAQ,EAAE,CAAC;YACZ,OAAiD,CAAC,aAAa,GAAG,QAAQ,CAAC;QAC9E,CAAC;QAED,IAAI,QAAQ,IAAI,cAAc,EAAE,CAAC;YAC/B,MAAM,kBAAkB,GAAG,OAAO,OAAO,CAAC,YAAY,CAAC,aAAa,KAAK,UAAU;gBACjF,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,aAAa,EAAE,CAAC,SAAS,EAAE;gBAClD,CAAC,CAAC,SAAS,CAAC;YACd,MAAM,eAAe,GAAG,cAAc,CAAC,kBAAkB,EAAE,IAAI,SAAS,CAAC;YAEzE,MAAM,KAAK,GAAG,kBAAkB,CAAC,wBAAwB,CACvD,SAAS,EACT,QAAQ,EACR;gBACE,cAAc;gBACd,kBAAkB;gBAClB,eAAe;gBACf,KAAK,EAAE;oBACL,QAAQ,EAAE,OAAO,CAAC,QAAQ;oBAC1B,WAAW,EAAE,OAAO,CAAC,WAAW;oBAChC,MAAM,EAAE,OAAO,CAAC,MAAM;iBACvB;aACF,CACF,CAAC;YACF,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;gBACpB,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,6CAA6C,EAAE;oBACvE,SAAS;oBACT,KAAK,EAAE,QAAQ,CAAC,iBAAiB,CAAC,MAAM;oBACxC,SAAS,EAAE,QAAQ,CAAC,aAAa,CAAC,MAAM;oBACxC,KAAK,EAAE,QAAQ,CAAC,aAAa,CAAC,MAAM;oBACpC,gBAAgB,EAAE,cAAc,CAAC,cAAc,EAAE,CAAC,QAAQ,EAAE;iBAC7D,CAAC,CAAC;YACL,CAAC;QACH,CAAC;aAAM,IAAI,cAAc,EAAE,CAAC;YAC1B,uEAAuE;YACvE,uEAAuE;YACvE,kCAAkC;YAClC,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC9C,kBAAkB,CAAC,WAAW,CAAC;oBAC7B,EAAE,EAAE,SAAS;oBACb,OAAO;oBACP,SAAS,EAAE,OAAO;oBAClB,QAAQ,EAAE,KAAK;oBACf,QAAQ,EAAE,EAAE;oBACZ,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;oBACrB,QAAQ,EAAE;wBACR,QAAQ,EAAE,OAAO,CAAC,QAAQ;wBAC1B,WAAW,EAAE,OAAO,CAAC,WAAW;wBAChC,MAAM,EAAE,OAAO,CAAC,MAAM;wBACtB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;wBAC1C,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;qBAC3C;iBACF,CAAC,CAAC;YACL,CAAC;YACD,kBAAkB,CAAC,kBAAkB,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;YACjE,IAAI,OAAO,OAAO,CAAC,YAAY,CAAC,qBAAqB,KAAK,UAAU,EAAE,CAAC;gBACrE,MAAM,UAAU,GAAG,OAAO,CAAC,YAAY,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;gBACvE,IAAI,UAAU;oBAAE,kBAAkB,CAAC,wBAAwB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;YACrF,CAAC;YACD,IAAI,OAAO,OAAO,CAAC,YAAY,CAAC,aAAa,KAAK,UAAU,EAAE,CAAC;gBAC7D,kBAAkB,CAAC,cAAc,CAAC,SAAS,EAAE,OAAO,CAAC,YAAY,CAAC,aAAa,EAAE,CAAC,CAAC;YACrF,CAAC;YACD,MAAM,eAAe,GAAG,cAAc,CAAC,kBAAkB,EAAE,CAAC;YAC5D,IAAI,eAAe;gBAAE,kBAAkB,CAAC,mBAAmB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;YACxF,kBAAkB,CAAC,iBAAiB,CAAC,SAAS,EAAE;gBAC9C,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,EAAE;gBAClD,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,EAAE;gBAC1C,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,EAAE;gBACpC,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,EAAE;gBACxC,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,EAAE;aACnD,CAAC,CAAC;QACL,CAAC;QAED,mEAAmE;QACnE,wEAAwE;QACxE,IAAI,cAAc,EAAE,CAAC;YACnB,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;gBAC3C,kBAAkB,CAAC,cAAc,CAAC,SAAS,EAAE;oBAC3C;wBACE,EAAE,EAAE,OAAO,SAAS,QAAQ,SAAS,OAAO;wBAC5C,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE,KAAK;wBACd,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,eAAe,IAAI,CAAC,CAAC;qBACtD;oBACD;wBACE,EAAE,EAAE,OAAO,SAAS,QAAQ,SAAS,YAAY;wBACjD,IAAI,EAAE,WAAW;wBACjB,OAAO,EAAE,CAAC,MAAM,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC;wBACtD,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;qBACtB;iBACF,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,MAAM,EAAE,CAAC;gBAChB,MAAM,EAAE,IAAI,CAAC,YAAY,EAAE,iCAAiC,EAAE;oBAC5D,KAAK,EAAG,MAAgB,CAAC,OAAO;iBACjC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,sEAAsE;QACtE,oEAAoE;QACpE,8CAA8C;QAC9C,MAAM,OAAO,GAAI,GAAa,CAAC,OAAO,CAAC;QACvC,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,iCAAiC,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;QACnF,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CAAC,gDAAgD,SAAS,eAAe,OAAO,EAAE,CAAC,CAAC;QAClG,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { RagStore } from '../ragStore';
|
|
2
|
+
import type { CodebaseRegistry } from '../codebase/codebaseRegistry';
|
|
3
|
+
import { PathSecurityGate } from '../codebase/pathSecurityGate';
|
|
4
|
+
export interface AospSourceIngestOptions {
|
|
5
|
+
maxChunkChars?: number;
|
|
6
|
+
pathPrefix?: string;
|
|
7
|
+
}
|
|
8
|
+
export interface AospSourceIngestResult {
|
|
9
|
+
codebaseId: string;
|
|
10
|
+
filesProcessed: number;
|
|
11
|
+
chunksAdded: number;
|
|
12
|
+
chunksSkipped: number;
|
|
13
|
+
blockedFileCount: number;
|
|
14
|
+
errors: Array<{
|
|
15
|
+
filePath: string;
|
|
16
|
+
reason: string;
|
|
17
|
+
}>;
|
|
18
|
+
}
|
|
19
|
+
export declare class AospSourceIngester {
|
|
20
|
+
private readonly store;
|
|
21
|
+
private readonly registry;
|
|
22
|
+
private readonly gate;
|
|
23
|
+
constructor(store: RagStore, registry: CodebaseRegistry, gate?: PathSecurityGate);
|
|
24
|
+
ingest(codebaseId: string, opts?: AospSourceIngestOptions): AospSourceIngestResult;
|
|
25
|
+
}
|
|
26
|
+
//# sourceMappingURL=aospSourceIngester.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"aospSourceIngester.d.ts","sourceRoot":"","sources":["../../../src/services/rag/aospSourceIngester.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAC1C,OAAO,KAAK,EAAC,gBAAgB,EAAc,MAAM,8BAA8B,CAAC;AAChF,OAAO,EAAC,gBAAgB,EAAyB,MAAM,8BAA8B,CAAC;AAUtF,MAAM,WAAW,uBAAuB;IACtC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,sBAAsB;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB,MAAM,EAAE,KAAK,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAC,CAAC,CAAC;CACnD;AAaD,qBAAa,kBAAkB;IAE3B,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,IAAI;gBAFJ,KAAK,EAAE,QAAQ,EACf,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,GAAE,gBAAyC;IAGlE,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,GAAE,uBAA4B,GAAG,sBAAsB;CA+EvF"}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
|
+
// Copyright (C) 2024-2026 Gracker (Chris)
|
|
4
|
+
// This file is part of SmartPerfetto. See LICENSE for details.
|
|
5
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
6
|
+
if (k2 === undefined) k2 = k;
|
|
7
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
8
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
9
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
10
|
+
}
|
|
11
|
+
Object.defineProperty(o, k2, desc);
|
|
12
|
+
}) : (function(o, m, k, k2) {
|
|
13
|
+
if (k2 === undefined) k2 = k;
|
|
14
|
+
o[k2] = m[k];
|
|
15
|
+
}));
|
|
16
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
17
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
18
|
+
}) : function(o, v) {
|
|
19
|
+
o["default"] = v;
|
|
20
|
+
});
|
|
21
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
22
|
+
var ownKeys = function(o) {
|
|
23
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
24
|
+
var ar = [];
|
|
25
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
26
|
+
return ar;
|
|
27
|
+
};
|
|
28
|
+
return ownKeys(o);
|
|
29
|
+
};
|
|
30
|
+
return function (mod) {
|
|
31
|
+
if (mod && mod.__esModule) return mod;
|
|
32
|
+
var result = {};
|
|
33
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
34
|
+
__setModuleDefault(result, mod);
|
|
35
|
+
return result;
|
|
36
|
+
};
|
|
37
|
+
})();
|
|
38
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
+
exports.AospSourceIngester = void 0;
|
|
40
|
+
const fs = __importStar(require("fs"));
|
|
41
|
+
const path = __importStar(require("path"));
|
|
42
|
+
const pathSecurityGate_1 = require("../codebase/pathSecurityGate");
|
|
43
|
+
const baseIngester_1 = require("./baseIngester");
|
|
44
|
+
const DEFAULT_MAX_CHUNK_CHARS = 2200;
|
|
45
|
+
function filterPreviewFiles(preview, ref, opts) {
|
|
46
|
+
const prefixes = [
|
|
47
|
+
...(ref.pathFilters ?? []),
|
|
48
|
+
...(opts.pathPrefix ? [opts.pathPrefix] : []),
|
|
49
|
+
].filter(Boolean).map(prefix => prefix.replace(/^\//, ''));
|
|
50
|
+
if (prefixes.length === 0)
|
|
51
|
+
return preview.acceptedFiles;
|
|
52
|
+
return preview.acceptedFiles.filter(file => prefixes.some(prefix => file.relativePath.startsWith(prefix)));
|
|
53
|
+
}
|
|
54
|
+
class AospSourceIngester {
|
|
55
|
+
constructor(store, registry, gate = new pathSecurityGate_1.PathSecurityGate()) {
|
|
56
|
+
this.store = store;
|
|
57
|
+
this.registry = registry;
|
|
58
|
+
this.gate = gate;
|
|
59
|
+
}
|
|
60
|
+
ingest(codebaseId, opts = {}) {
|
|
61
|
+
const ref = this.registry.get(codebaseId);
|
|
62
|
+
if (!ref)
|
|
63
|
+
throw new Error(`Codebase '${codebaseId}' not found`);
|
|
64
|
+
if (ref.kind !== 'aosp') {
|
|
65
|
+
throw new Error(`Codebase '${codebaseId}' is kind=${ref.kind}; AOSP ingestion requires aosp`);
|
|
66
|
+
}
|
|
67
|
+
if (!ref.licenseTag) {
|
|
68
|
+
throw new Error(`AOSP codebase '${codebaseId}' requires licenseTag`);
|
|
69
|
+
}
|
|
70
|
+
const preview = this.gate.preview(ref.rootRealpath);
|
|
71
|
+
if (preview.blocked) {
|
|
72
|
+
this.registry.updateIngestStatus(codebaseId, {
|
|
73
|
+
lastIngestStatus: 'blocked_by_security',
|
|
74
|
+
lastIngestAt: Date.now(),
|
|
75
|
+
lastIngestError: preview.blockedReason,
|
|
76
|
+
blockedFileCount: preview.skippedFiles.length,
|
|
77
|
+
});
|
|
78
|
+
return {
|
|
79
|
+
codebaseId,
|
|
80
|
+
filesProcessed: 0,
|
|
81
|
+
chunksAdded: 0,
|
|
82
|
+
chunksSkipped: 0,
|
|
83
|
+
blockedFileCount: preview.skippedFiles.length,
|
|
84
|
+
errors: [{ filePath: ref.displayName, reason: preview.blockedReason ?? 'blocked' }],
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
const result = {
|
|
88
|
+
codebaseId,
|
|
89
|
+
filesProcessed: 0,
|
|
90
|
+
chunksAdded: 0,
|
|
91
|
+
chunksSkipped: 0,
|
|
92
|
+
blockedFileCount: preview.skippedFiles.length,
|
|
93
|
+
errors: [],
|
|
94
|
+
};
|
|
95
|
+
const maxChars = opts.maxChunkChars ?? DEFAULT_MAX_CHUNK_CHARS;
|
|
96
|
+
for (const file of filterPreviewFiles(preview, ref, opts)) {
|
|
97
|
+
result.filesProcessed++;
|
|
98
|
+
try {
|
|
99
|
+
const content = fs.readFileSync(path.join(ref.rootRealpath, file.relativePath), 'utf-8');
|
|
100
|
+
const chunks = (0, baseIngester_1.chunkSourceBySymbols)(content, maxChars);
|
|
101
|
+
if (chunks.length === 0) {
|
|
102
|
+
result.chunksSkipped++;
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
for (const chunk of chunks) {
|
|
106
|
+
this.store.addChunk({
|
|
107
|
+
chunkId: (0, baseIngester_1.stableChunkId)([codebaseId, ref.indexGeneration, file.relativePath, chunk.startLine]),
|
|
108
|
+
kind: 'aosp',
|
|
109
|
+
uri: `codebase://${codebaseId}/${file.relativePath}`,
|
|
110
|
+
title: file.relativePath.split('/').pop(),
|
|
111
|
+
snippet: chunk.text,
|
|
112
|
+
tokenCount: (0, baseIngester_1.estimateTokenCount)(chunk.text),
|
|
113
|
+
license: ref.licenseTag,
|
|
114
|
+
indexedAt: Date.now(),
|
|
115
|
+
filePath: file.relativePath,
|
|
116
|
+
lineRange: { start: chunk.startLine, end: chunk.endLine },
|
|
117
|
+
...(chunk.symbol ? { symbol: chunk.symbol } : {}),
|
|
118
|
+
language: (0, baseIngester_1.languageForPath)(file.relativePath),
|
|
119
|
+
...(ref.commitHash ? { commitHash: ref.commitHash } : {}),
|
|
120
|
+
...(ref.buildId ? { buildId: ref.buildId } : {}),
|
|
121
|
+
codebaseId,
|
|
122
|
+
registryOrigin: 'codebase_registry',
|
|
123
|
+
});
|
|
124
|
+
result.chunksAdded++;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
catch (error) {
|
|
128
|
+
result.chunksSkipped++;
|
|
129
|
+
result.errors.push({ filePath: file.relativePath, reason: error instanceof Error ? error.message : String(error) });
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
this.registry.updateIngestStatus(codebaseId, {
|
|
133
|
+
lastIngestStatus: result.errors.length > 0 ? 'partial' : 'ok',
|
|
134
|
+
lastIngestAt: Date.now(),
|
|
135
|
+
lastIngestError: result.errors[0]?.reason,
|
|
136
|
+
chunkCount: result.chunksAdded,
|
|
137
|
+
blockedFileCount: result.blockedFileCount,
|
|
138
|
+
});
|
|
139
|
+
return result;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
exports.AospSourceIngester = AospSourceIngester;
|
|
143
|
+
//# sourceMappingURL=aospSourceIngester.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"aospSourceIngester.js","sourceRoot":"","sources":["../../../src/services/rag/aospSourceIngester.ts"],"names":[],"mappings":";AAAA,6CAA6C;AAC7C,0CAA0C;AAC1C,+DAA+D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE/D,uCAAyB;AACzB,2CAA6B;AAI7B,mEAAsF;AACtF,iDAKwB;AAExB,MAAM,uBAAuB,GAAG,IAAI,CAAC;AAgBrC,SAAS,kBAAkB,CAAC,OAA0B,EAAE,GAAgB,EAAE,IAA6B;IACrG,MAAM,QAAQ,GAAG;QACf,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC;QAC1B,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9C,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;IAC3D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC,aAAa,CAAC;IACxD,OAAO,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CACzC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAC9D,CAAC;AACJ,CAAC;AAED,MAAa,kBAAkB;IAC7B,YACmB,KAAe,EACf,QAA0B,EAC1B,OAAyB,IAAI,mCAAgB,EAAE;QAF/C,UAAK,GAAL,KAAK,CAAU;QACf,aAAQ,GAAR,QAAQ,CAAkB;QAC1B,SAAI,GAAJ,IAAI,CAA2C;IAC/D,CAAC;IAEJ,MAAM,CAAC,UAAkB,EAAE,OAAgC,EAAE;QAC3D,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,CAAC,GAAG;YAAE,MAAM,IAAI,KAAK,CAAC,aAAa,UAAU,aAAa,CAAC,CAAC;QAChE,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,aAAa,UAAU,aAAa,GAAG,CAAC,IAAI,gCAAgC,CAAC,CAAC;QAChG,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,kBAAkB,UAAU,uBAAuB,CAAC,CAAC;QACvE,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACpD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,UAAU,EAAE;gBAC3C,gBAAgB,EAAE,qBAAqB;gBACvC,YAAY,EAAE,IAAI,CAAC,GAAG,EAAE;gBACxB,eAAe,EAAE,OAAO,CAAC,aAAa;gBACtC,gBAAgB,EAAE,OAAO,CAAC,YAAY,CAAC,MAAM;aAC9C,CAAC,CAAC;YACH,OAAO;gBACL,UAAU;gBACV,cAAc,EAAE,CAAC;gBACjB,WAAW,EAAE,CAAC;gBACd,aAAa,EAAE,CAAC;gBAChB,gBAAgB,EAAE,OAAO,CAAC,YAAY,CAAC,MAAM;gBAC7C,MAAM,EAAE,CAAC,EAAC,QAAQ,EAAE,GAAG,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,aAAa,IAAI,SAAS,EAAC,CAAC;aAClF,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAA2B;YACrC,UAAU;YACV,cAAc,EAAE,CAAC;YACjB,WAAW,EAAE,CAAC;YACd,aAAa,EAAE,CAAC;YAChB,gBAAgB,EAAE,OAAO,CAAC,YAAY,CAAC,MAAM;YAC7C,MAAM,EAAE,EAAE;SACX,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,IAAI,uBAAuB,CAAC;QAC/D,KAAK,MAAM,IAAI,IAAI,kBAAkB,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,CAAC;YAC1D,MAAM,CAAC,cAAc,EAAE,CAAC;YACxB,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,IAAI,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC,CAAC;gBACzF,MAAM,MAAM,GAAG,IAAA,mCAAoB,EAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;gBACvD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACxB,MAAM,CAAC,aAAa,EAAE,CAAC;oBACvB,SAAS;gBACX,CAAC;gBACD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;oBAC3B,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;wBAClB,OAAO,EAAE,IAAA,4BAAa,EAAC,CAAC,UAAU,EAAE,GAAG,CAAC,eAAe,EAAE,IAAI,CAAC,YAAY,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;wBAC7F,IAAI,EAAE,MAAM;wBACZ,GAAG,EAAE,cAAc,UAAU,IAAI,IAAI,CAAC,YAAY,EAAE;wBACpD,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE;wBACzC,OAAO,EAAE,KAAK,CAAC,IAAI;wBACnB,UAAU,EAAE,IAAA,iCAAkB,EAAC,KAAK,CAAC,IAAI,CAAC;wBAC1C,OAAO,EAAE,GAAG,CAAC,UAAU;wBACvB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;wBACrB,QAAQ,EAAE,IAAI,CAAC,YAAY;wBAC3B,SAAS,EAAE,EAAC,KAAK,EAAE,KAAK,CAAC,SAAS,EAAE,GAAG,EAAE,KAAK,CAAC,OAAO,EAAC;wBACvD,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAC,CAAC,CAAC,CAAC,EAAE,CAAC;wBAC/C,QAAQ,EAAE,IAAA,8BAAe,EAAC,IAAI,CAAC,YAAY,CAAC;wBAC5C,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAC,UAAU,EAAE,GAAG,CAAC,UAAU,EAAC,CAAC,CAAC,CAAC,EAAE,CAAC;wBACvD,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAC,OAAO,EAAE,GAAG,CAAC,OAAO,EAAC,CAAC,CAAC,CAAC,EAAE,CAAC;wBAC9C,UAAU;wBACV,cAAc,EAAE,mBAAmB;qBACpC,CAAC,CAAC;oBACH,MAAM,CAAC,WAAW,EAAE,CAAC;gBACvB,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,aAAa,EAAE,CAAC;gBACvB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAC,QAAQ,EAAE,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAC,CAAC,CAAC;YACpH,CAAC;QACH,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,UAAU,EAAE;YAC3C,gBAAgB,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;YAC7D,YAAY,EAAE,IAAI,CAAC,GAAG,EAAE;YACxB,eAAe,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM;YACzC,UAAU,EAAE,MAAM,CAAC,WAAW;YAC9B,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;SAC1C,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAtFD,gDAsFC"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { RagStore } from '../ragStore';
|
|
2
|
+
import type { CodebaseRegistry } from '../codebase/codebaseRegistry';
|
|
3
|
+
import { PathSecurityGate } from '../codebase/pathSecurityGate';
|
|
4
|
+
import { chunkSourceBySymbols, detectSourceSymbol, languageForPath } from './baseIngester';
|
|
5
|
+
export interface AppSourceIngestOptions {
|
|
6
|
+
maxChunkChars?: number;
|
|
7
|
+
pathPrefix?: string;
|
|
8
|
+
}
|
|
9
|
+
export interface AppSourceIngestError {
|
|
10
|
+
filePath: string;
|
|
11
|
+
reason: string;
|
|
12
|
+
}
|
|
13
|
+
export interface AppSourceIngestResult {
|
|
14
|
+
codebaseId: string;
|
|
15
|
+
filesProcessed: number;
|
|
16
|
+
chunksAdded: number;
|
|
17
|
+
chunksSkipped: number;
|
|
18
|
+
blockedFileCount: number;
|
|
19
|
+
redactionHitCount: number;
|
|
20
|
+
errors: AppSourceIngestError[];
|
|
21
|
+
}
|
|
22
|
+
declare function makeChunkId(codebaseId: string, indexGeneration: number, relativePath: string, startLine: number): string;
|
|
23
|
+
export declare class AppSourceIngester {
|
|
24
|
+
private readonly store;
|
|
25
|
+
private readonly registry;
|
|
26
|
+
private readonly gate;
|
|
27
|
+
constructor(store: RagStore, registry: CodebaseRegistry, gate?: PathSecurityGate);
|
|
28
|
+
ingest(codebaseId: string, opts?: AppSourceIngestOptions): AppSourceIngestResult;
|
|
29
|
+
}
|
|
30
|
+
export declare const __TEST_ONLY__: {
|
|
31
|
+
chunkSource: typeof chunkSourceBySymbols;
|
|
32
|
+
detectSymbol: typeof detectSourceSymbol;
|
|
33
|
+
makeChunkId: typeof makeChunkId;
|
|
34
|
+
languageFor: typeof languageForPath;
|
|
35
|
+
};
|
|
36
|
+
export {};
|
|
37
|
+
//# sourceMappingURL=appSourceIngester.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"appSourceIngester.d.ts","sourceRoot":"","sources":["../../../src/services/rag/appSourceIngester.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AAE1C,OAAO,KAAK,EAAC,gBAAgB,EAAc,MAAM,8BAA8B,CAAC;AAChF,OAAO,EAAC,gBAAgB,EAAyB,MAAM,8BAA8B,CAAC;AACtF,OAAO,EACL,oBAAoB,EACpB,kBAAkB,EAElB,eAAe,EAChB,MAAM,gBAAgB,CAAC;AAIxB,MAAM,WAAW,sBAAsB;IACrC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,qBAAqB;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,MAAM,EAAE,oBAAoB,EAAE,CAAC;CAChC;AAED,iBAAS,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAKjH;AAeD,qBAAa,iBAAiB;IAE1B,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,IAAI;gBAFJ,KAAK,EAAE,QAAQ,EACf,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,GAAE,gBAAyC;IAGlE,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,GAAE,sBAA2B,GAAG,qBAAqB;CA2FrF;AAED,eAAO,MAAM,aAAa;;;;;CAKzB,CAAC"}
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
|
+
// Copyright (C) 2024-2026 Gracker (Chris)
|
|
4
|
+
// This file is part of SmartPerfetto. See LICENSE for details.
|
|
5
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
6
|
+
if (k2 === undefined) k2 = k;
|
|
7
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
8
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
9
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
10
|
+
}
|
|
11
|
+
Object.defineProperty(o, k2, desc);
|
|
12
|
+
}) : (function(o, m, k, k2) {
|
|
13
|
+
if (k2 === undefined) k2 = k;
|
|
14
|
+
o[k2] = m[k];
|
|
15
|
+
}));
|
|
16
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
17
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
18
|
+
}) : function(o, v) {
|
|
19
|
+
o["default"] = v;
|
|
20
|
+
});
|
|
21
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
22
|
+
var ownKeys = function(o) {
|
|
23
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
24
|
+
var ar = [];
|
|
25
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
26
|
+
return ar;
|
|
27
|
+
};
|
|
28
|
+
return ownKeys(o);
|
|
29
|
+
};
|
|
30
|
+
return function (mod) {
|
|
31
|
+
if (mod && mod.__esModule) return mod;
|
|
32
|
+
var result = {};
|
|
33
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
34
|
+
__setModuleDefault(result, mod);
|
|
35
|
+
return result;
|
|
36
|
+
};
|
|
37
|
+
})();
|
|
38
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
+
exports.__TEST_ONLY__ = exports.AppSourceIngester = void 0;
|
|
40
|
+
const crypto_1 = require("crypto");
|
|
41
|
+
const fs = __importStar(require("fs"));
|
|
42
|
+
const path = __importStar(require("path"));
|
|
43
|
+
const secretPatterns_1 = require("../security/secretPatterns");
|
|
44
|
+
const pathSecurityGate_1 = require("../codebase/pathSecurityGate");
|
|
45
|
+
const baseIngester_1 = require("./baseIngester");
|
|
46
|
+
const DEFAULT_MAX_CHUNK_CHARS = 2200;
|
|
47
|
+
function makeChunkId(codebaseId, indexGeneration, relativePath, startLine) {
|
|
48
|
+
return (0, crypto_1.createHash)('sha256')
|
|
49
|
+
.update(`${codebaseId}|${indexGeneration}|${relativePath}|${startLine}`)
|
|
50
|
+
.digest('hex')
|
|
51
|
+
.slice(0, 16);
|
|
52
|
+
}
|
|
53
|
+
const chunkSource = baseIngester_1.chunkSourceBySymbols;
|
|
54
|
+
function filterPreviewFiles(preview, ref, opts) {
|
|
55
|
+
const prefixes = [
|
|
56
|
+
...(ref.pathFilters ?? []),
|
|
57
|
+
...(opts.pathPrefix ? [opts.pathPrefix] : []),
|
|
58
|
+
].filter(Boolean);
|
|
59
|
+
if (prefixes.length === 0)
|
|
60
|
+
return preview.acceptedFiles;
|
|
61
|
+
return preview.acceptedFiles.filter(file => prefixes.some(prefix => file.relativePath.startsWith(prefix.replace(/^\//, ''))));
|
|
62
|
+
}
|
|
63
|
+
class AppSourceIngester {
|
|
64
|
+
constructor(store, registry, gate = new pathSecurityGate_1.PathSecurityGate()) {
|
|
65
|
+
this.store = store;
|
|
66
|
+
this.registry = registry;
|
|
67
|
+
this.gate = gate;
|
|
68
|
+
}
|
|
69
|
+
ingest(codebaseId, opts = {}) {
|
|
70
|
+
const ref = this.registry.get(codebaseId);
|
|
71
|
+
if (!ref) {
|
|
72
|
+
throw new Error(`Codebase '${codebaseId}' not found`);
|
|
73
|
+
}
|
|
74
|
+
if (ref.kind !== 'app_source') {
|
|
75
|
+
throw new Error(`Codebase '${codebaseId}' is kind=${ref.kind}; app source ingestion requires app_source`);
|
|
76
|
+
}
|
|
77
|
+
const preview = this.gate.preview(ref.rootRealpath);
|
|
78
|
+
if (preview.blocked) {
|
|
79
|
+
this.registry.updateIngestStatus(codebaseId, {
|
|
80
|
+
lastIngestStatus: 'blocked_by_security',
|
|
81
|
+
lastIngestAt: Date.now(),
|
|
82
|
+
lastIngestError: preview.blockedReason,
|
|
83
|
+
blockedFileCount: preview.skippedFiles.length,
|
|
84
|
+
});
|
|
85
|
+
return {
|
|
86
|
+
codebaseId,
|
|
87
|
+
filesProcessed: 0,
|
|
88
|
+
chunksAdded: 0,
|
|
89
|
+
chunksSkipped: 0,
|
|
90
|
+
blockedFileCount: preview.skippedFiles.length,
|
|
91
|
+
redactionHitCount: 0,
|
|
92
|
+
errors: [{ filePath: ref.displayName, reason: preview.blockedReason ?? 'blocked' }],
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
const maxChars = opts.maxChunkChars ?? DEFAULT_MAX_CHUNK_CHARS;
|
|
96
|
+
const result = {
|
|
97
|
+
codebaseId,
|
|
98
|
+
filesProcessed: 0,
|
|
99
|
+
chunksAdded: 0,
|
|
100
|
+
chunksSkipped: 0,
|
|
101
|
+
blockedFileCount: preview.skippedFiles.length,
|
|
102
|
+
redactionHitCount: 0,
|
|
103
|
+
errors: [],
|
|
104
|
+
};
|
|
105
|
+
for (const file of filterPreviewFiles(preview, ref, opts)) {
|
|
106
|
+
result.filesProcessed++;
|
|
107
|
+
try {
|
|
108
|
+
const absolutePath = path.join(ref.rootRealpath, file.relativePath);
|
|
109
|
+
const content = fs.readFileSync(absolutePath, 'utf-8');
|
|
110
|
+
const chunks = chunkSource(content, maxChars);
|
|
111
|
+
if (chunks.length === 0) {
|
|
112
|
+
result.chunksSkipped++;
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
for (const chunk of chunks) {
|
|
116
|
+
const redaction = (0, secretPatterns_1.redactSecrets)(chunk.text);
|
|
117
|
+
result.redactionHitCount += redaction.redactedCount;
|
|
118
|
+
this.store.addChunk({
|
|
119
|
+
chunkId: makeChunkId(codebaseId, ref.indexGeneration, file.relativePath, chunk.startLine),
|
|
120
|
+
kind: 'app_source',
|
|
121
|
+
uri: `codebase://${codebaseId}/${file.relativePath}`,
|
|
122
|
+
title: file.relativePath.split('/').pop(),
|
|
123
|
+
snippet: chunk.text,
|
|
124
|
+
tokenCount: (0, baseIngester_1.estimateTokenCount)(chunk.text),
|
|
125
|
+
indexedAt: Date.now(),
|
|
126
|
+
filePath: file.relativePath,
|
|
127
|
+
lineRange: { start: chunk.startLine, end: chunk.endLine },
|
|
128
|
+
...(chunk.symbol ? { symbol: chunk.symbol } : {}),
|
|
129
|
+
language: (0, baseIngester_1.languageForPath)(file.relativePath),
|
|
130
|
+
...(ref.commitHash ? { commitHash: ref.commitHash } : {}),
|
|
131
|
+
...(ref.vendor ? { vendor: ref.vendor } : {}),
|
|
132
|
+
...(ref.buildId ? { buildId: ref.buildId } : {}),
|
|
133
|
+
codebaseId,
|
|
134
|
+
registryOrigin: 'codebase_registry',
|
|
135
|
+
});
|
|
136
|
+
result.chunksAdded++;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
catch (error) {
|
|
140
|
+
result.chunksSkipped++;
|
|
141
|
+
result.errors.push({
|
|
142
|
+
filePath: file.relativePath,
|
|
143
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
this.registry.updateIngestStatus(codebaseId, {
|
|
148
|
+
lastIngestStatus: result.errors.length > 0 ? 'partial' : 'ok',
|
|
149
|
+
lastIngestAt: Date.now(),
|
|
150
|
+
lastIngestError: result.errors[0]?.reason,
|
|
151
|
+
chunkCount: result.chunksAdded,
|
|
152
|
+
blockedFileCount: result.blockedFileCount,
|
|
153
|
+
redactionHitCount: result.redactionHitCount,
|
|
154
|
+
});
|
|
155
|
+
return result;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
exports.AppSourceIngester = AppSourceIngester;
|
|
159
|
+
exports.__TEST_ONLY__ = {
|
|
160
|
+
chunkSource,
|
|
161
|
+
detectSymbol: baseIngester_1.detectSourceSymbol,
|
|
162
|
+
makeChunkId,
|
|
163
|
+
languageFor: baseIngester_1.languageForPath,
|
|
164
|
+
};
|
|
165
|
+
//# sourceMappingURL=appSourceIngester.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"appSourceIngester.js","sourceRoot":"","sources":["../../../src/services/rag/appSourceIngester.ts"],"names":[],"mappings":";AAAA,6CAA6C;AAC7C,0CAA0C;AAC1C,+DAA+D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE/D,mCAAkC;AAClC,uCAAyB;AACzB,2CAA6B;AAG7B,+DAAyD;AAEzD,mEAAsF;AACtF,iDAKwB;AAExB,MAAM,uBAAuB,GAAG,IAAI,CAAC;AAsBrC,SAAS,WAAW,CAAC,UAAkB,EAAE,eAAuB,EAAE,YAAoB,EAAE,SAAiB;IACvG,OAAO,IAAA,mBAAU,EAAC,QAAQ,CAAC;SACxB,MAAM,CAAC,GAAG,UAAU,IAAI,eAAe,IAAI,YAAY,IAAI,SAAS,EAAE,CAAC;SACvE,MAAM,CAAC,KAAK,CAAC;SACb,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,WAAW,GAAG,mCAAoB,CAAC;AAEzC,SAAS,kBAAkB,CAAC,OAA0B,EAAE,GAAgB,EAAE,IAA4B;IACpG,MAAM,QAAQ,GAAG;QACf,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC;QAC1B,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9C,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAClB,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC,aAAa,CAAC;IACxD,OAAO,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CACzC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CACjF,CAAC;AACJ,CAAC;AAED,MAAa,iBAAiB;IAC5B,YACmB,KAAe,EACf,QAA0B,EAC1B,OAAyB,IAAI,mCAAgB,EAAE;QAF/C,UAAK,GAAL,KAAK,CAAU;QACf,aAAQ,GAAR,QAAQ,CAAkB;QAC1B,SAAI,GAAJ,IAAI,CAA2C;IAC/D,CAAC;IAEJ,MAAM,CAAC,UAAkB,EAAE,OAA+B,EAAE;QAC1D,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,aAAa,UAAU,aAAa,CAAC,CAAC;QACxD,CAAC;QACD,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,aAAa,UAAU,aAAa,GAAG,CAAC,IAAI,4CAA4C,CAAC,CAAC;QAC5G,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACpD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,UAAU,EAAE;gBAC3C,gBAAgB,EAAE,qBAAqB;gBACvC,YAAY,EAAE,IAAI,CAAC,GAAG,EAAE;gBACxB,eAAe,EAAE,OAAO,CAAC,aAAa;gBACtC,gBAAgB,EAAE,OAAO,CAAC,YAAY,CAAC,MAAM;aAC9C,CAAC,CAAC;YACH,OAAO;gBACL,UAAU;gBACV,cAAc,EAAE,CAAC;gBACjB,WAAW,EAAE,CAAC;gBACd,aAAa,EAAE,CAAC;gBAChB,gBAAgB,EAAE,OAAO,CAAC,YAAY,CAAC,MAAM;gBAC7C,iBAAiB,EAAE,CAAC;gBACpB,MAAM,EAAE,CAAC,EAAC,QAAQ,EAAE,GAAG,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,aAAa,IAAI,SAAS,EAAC,CAAC;aAClF,CAAC;QACJ,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,IAAI,uBAAuB,CAAC;QAC/D,MAAM,MAAM,GAA0B;YACpC,UAAU;YACV,cAAc,EAAE,CAAC;YACjB,WAAW,EAAE,CAAC;YACd,aAAa,EAAE,CAAC;YAChB,gBAAgB,EAAE,OAAO,CAAC,YAAY,CAAC,MAAM;YAC7C,iBAAiB,EAAE,CAAC;YACpB,MAAM,EAAE,EAAE;SACX,CAAC;QAEF,KAAK,MAAM,IAAI,IAAI,kBAAkB,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,CAAC;YAC1D,MAAM,CAAC,cAAc,EAAE,CAAC;YACxB,IAAI,CAAC;gBACH,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;gBACpE,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;gBACvD,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;gBAC9C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACxB,MAAM,CAAC,aAAa,EAAE,CAAC;oBACvB,SAAS;gBACX,CAAC;gBACD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;oBAC3B,MAAM,SAAS,GAAG,IAAA,8BAAa,EAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBAC5C,MAAM,CAAC,iBAAiB,IAAI,SAAS,CAAC,aAAa,CAAC;oBACpD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;wBAClB,OAAO,EAAE,WAAW,CAAC,UAAU,EAAE,GAAG,CAAC,eAAe,EAAE,IAAI,CAAC,YAAY,EAAE,KAAK,CAAC,SAAS,CAAC;wBACzF,IAAI,EAAE,YAAY;wBAClB,GAAG,EAAE,cAAc,UAAU,IAAI,IAAI,CAAC,YAAY,EAAE;wBACpD,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE;wBACzC,OAAO,EAAE,KAAK,CAAC,IAAI;wBACnB,UAAU,EAAE,IAAA,iCAAkB,EAAC,KAAK,CAAC,IAAI,CAAC;wBAC1C,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;wBACrB,QAAQ,EAAE,IAAI,CAAC,YAAY;wBAC3B,SAAS,EAAE,EAAC,KAAK,EAAE,KAAK,CAAC,SAAS,EAAE,GAAG,EAAE,KAAK,CAAC,OAAO,EAAC;wBACvD,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAC,CAAC,CAAC,CAAC,EAAE,CAAC;wBAC/C,QAAQ,EAAE,IAAA,8BAAe,EAAC,IAAI,CAAC,YAAY,CAAC;wBAC5C,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAC,UAAU,EAAE,GAAG,CAAC,UAAU,EAAC,CAAC,CAAC,CAAC,EAAE,CAAC;wBACvD,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAC,CAAC,CAAC,CAAC,EAAE,CAAC;wBAC3C,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAC,OAAO,EAAE,GAAG,CAAC,OAAO,EAAC,CAAC,CAAC,CAAC,EAAE,CAAC;wBAC9C,UAAU;wBACV,cAAc,EAAE,mBAAmB;qBACpC,CAAC,CAAC;oBACH,MAAM,CAAC,WAAW,EAAE,CAAC;gBACvB,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,aAAa,EAAE,CAAC;gBACvB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;oBACjB,QAAQ,EAAE,IAAI,CAAC,YAAY;oBAC3B,MAAM,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;iBAC/D,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,UAAU,EAAE;YAC3C,gBAAgB,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;YAC7D,YAAY,EAAE,IAAI,CAAC,GAAG,EAAE;YACxB,eAAe,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM;YACzC,UAAU,EAAE,MAAM,CAAC,WAAW;YAC9B,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;YACzC,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;SAC5C,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAlGD,8CAkGC;AAEY,QAAA,aAAa,GAAG;IAC3B,WAAW;IACX,YAAY,EAAE,iCAAkB;IAChC,WAAW;IACX,WAAW,EAAE,8BAAe;CAC7B,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { RagChunk } from '../../types/sparkContracts';
|
|
2
|
+
export interface SourceChunk {
|
|
3
|
+
text: string;
|
|
4
|
+
startLine: number;
|
|
5
|
+
endLine: number;
|
|
6
|
+
symbol?: string;
|
|
7
|
+
}
|
|
8
|
+
export interface OffsetChunk {
|
|
9
|
+
text: string;
|
|
10
|
+
offset: number;
|
|
11
|
+
}
|
|
12
|
+
export declare function estimateTokenCount(text: string): number;
|
|
13
|
+
export declare function stableChunkId(parts: readonly unknown[], length?: number): string;
|
|
14
|
+
export declare function languageForPath(relativePath: string): NonNullable<RagChunk['language']>;
|
|
15
|
+
export declare function detectSourceSymbol(line: string): string | undefined;
|
|
16
|
+
export declare function chunkSourceBySymbols(content: string, maxChars: number): SourceChunk[];
|
|
17
|
+
export declare function chunkTextByParagraphs(text: string, maxChars: number): OffsetChunk[];
|
|
18
|
+
//# sourceMappingURL=baseIngester.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"baseIngester.d.ts","sourceRoot":"","sources":["../../../src/services/rag/baseIngester.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,4BAA4B,CAAC;AAEzD,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEvD;AAED,wBAAgB,aAAa,CAAC,KAAK,EAAE,SAAS,OAAO,EAAE,EAAE,MAAM,SAAK,GAAG,MAAM,CAK5E;AAED,wBAAgB,eAAe,CAAC,YAAY,EAAE,MAAM,GAAG,WAAW,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAUvF;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CASnE;AAED,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,WAAW,EAAE,CA2ErF;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,WAAW,EAAE,CA0BnF"}
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
3
|
+
// Copyright (C) 2024-2026 Gracker (Chris)
|
|
4
|
+
// This file is part of SmartPerfetto. See LICENSE for details.
|
|
5
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
6
|
+
if (k2 === undefined) k2 = k;
|
|
7
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
8
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
9
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
10
|
+
}
|
|
11
|
+
Object.defineProperty(o, k2, desc);
|
|
12
|
+
}) : (function(o, m, k, k2) {
|
|
13
|
+
if (k2 === undefined) k2 = k;
|
|
14
|
+
o[k2] = m[k];
|
|
15
|
+
}));
|
|
16
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
17
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
18
|
+
}) : function(o, v) {
|
|
19
|
+
o["default"] = v;
|
|
20
|
+
});
|
|
21
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
22
|
+
var ownKeys = function(o) {
|
|
23
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
24
|
+
var ar = [];
|
|
25
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
26
|
+
return ar;
|
|
27
|
+
};
|
|
28
|
+
return ownKeys(o);
|
|
29
|
+
};
|
|
30
|
+
return function (mod) {
|
|
31
|
+
if (mod && mod.__esModule) return mod;
|
|
32
|
+
var result = {};
|
|
33
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
34
|
+
__setModuleDefault(result, mod);
|
|
35
|
+
return result;
|
|
36
|
+
};
|
|
37
|
+
})();
|
|
38
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
+
exports.estimateTokenCount = estimateTokenCount;
|
|
40
|
+
exports.stableChunkId = stableChunkId;
|
|
41
|
+
exports.languageForPath = languageForPath;
|
|
42
|
+
exports.detectSourceSymbol = detectSourceSymbol;
|
|
43
|
+
exports.chunkSourceBySymbols = chunkSourceBySymbols;
|
|
44
|
+
exports.chunkTextByParagraphs = chunkTextByParagraphs;
|
|
45
|
+
const crypto_1 = require("crypto");
|
|
46
|
+
const path = __importStar(require("path"));
|
|
47
|
+
function estimateTokenCount(text) {
|
|
48
|
+
return Math.max(1, Math.round(text.length / 4));
|
|
49
|
+
}
|
|
50
|
+
function stableChunkId(parts, length = 16) {
|
|
51
|
+
return (0, crypto_1.createHash)('sha256')
|
|
52
|
+
.update(parts.map(part => String(part)).join('|'))
|
|
53
|
+
.digest('hex')
|
|
54
|
+
.slice(0, length);
|
|
55
|
+
}
|
|
56
|
+
function languageForPath(relativePath) {
|
|
57
|
+
const ext = path.extname(relativePath);
|
|
58
|
+
if (ext === '.kt' || ext === '.kts')
|
|
59
|
+
return 'kotlin';
|
|
60
|
+
if (ext === '.java')
|
|
61
|
+
return 'java';
|
|
62
|
+
if (ext === '.c')
|
|
63
|
+
return 'c';
|
|
64
|
+
if (ext === '.cc' || ext === '.cpp' || ext === '.cxx' || ext === '.h' || ext === '.hpp')
|
|
65
|
+
return 'cpp';
|
|
66
|
+
if (ext === '.rs')
|
|
67
|
+
return 'rust';
|
|
68
|
+
if (ext === '.go')
|
|
69
|
+
return 'go';
|
|
70
|
+
if (ext === '.py')
|
|
71
|
+
return 'py';
|
|
72
|
+
return 'unknown';
|
|
73
|
+
}
|
|
74
|
+
function detectSourceSymbol(line) {
|
|
75
|
+
const classMatch = line.match(/\b(?:class|object|interface|enum\s+class|struct)\s+([A-Za-z_][A-Za-z0-9_]*)/);
|
|
76
|
+
if (classMatch?.[1])
|
|
77
|
+
return classMatch[1];
|
|
78
|
+
const kotlinFun = line.match(/\bfun\s+([A-Za-z_][A-Za-z0-9_]*)\s*\(/);
|
|
79
|
+
if (kotlinFun?.[1])
|
|
80
|
+
return kotlinFun[1];
|
|
81
|
+
const cLikeFunction = line.match(/\b(?:[A-Za-z_][A-Za-z0-9_:<>~*&\s]+)\s+([A-Za-z_~][A-Za-z0-9_:~]*)\s*\([^;]*\)\s*(?:const\s*)?(?:noexcept\s*)?(?:override\s*)?\{/);
|
|
82
|
+
if (cLikeFunction?.[1])
|
|
83
|
+
return cLikeFunction[1].split('::').pop();
|
|
84
|
+
const javaMethod = line.match(/\b(?:public|private|protected|internal|static|final|suspend|override|\s)+[A-Za-z0-9_<>,.?[\]\s]+\s+([A-Za-z_][A-Za-z0-9_]*)\s*\([^;]*\)\s*\{/);
|
|
85
|
+
return javaMethod?.[1];
|
|
86
|
+
}
|
|
87
|
+
function chunkSourceBySymbols(content, maxChars) {
|
|
88
|
+
const normalized = content.replace(/\r\n?/g, '\n');
|
|
89
|
+
if (normalized.trim().length === 0)
|
|
90
|
+
return [];
|
|
91
|
+
const lines = normalized.split('\n');
|
|
92
|
+
const boundaries = new Set([0]);
|
|
93
|
+
for (let i = 0; i < lines.length; i++) {
|
|
94
|
+
if (detectSourceSymbol(lines[i]))
|
|
95
|
+
boundaries.add(i);
|
|
96
|
+
}
|
|
97
|
+
boundaries.add(lines.length);
|
|
98
|
+
const sorted = Array.from(boundaries).sort((a, b) => a - b);
|
|
99
|
+
const sections = [];
|
|
100
|
+
for (let i = 0; i < sorted.length - 1; i++) {
|
|
101
|
+
const start = sorted[i];
|
|
102
|
+
const end = sorted[i + 1];
|
|
103
|
+
if (end <= start)
|
|
104
|
+
continue;
|
|
105
|
+
const text = lines.slice(start, end).join('\n').trim();
|
|
106
|
+
if (!text)
|
|
107
|
+
continue;
|
|
108
|
+
sections.push({
|
|
109
|
+
text,
|
|
110
|
+
startLine: start + 1,
|
|
111
|
+
endLine: end,
|
|
112
|
+
symbol: detectSourceSymbol(lines[start]),
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
const out = [];
|
|
116
|
+
for (const section of sections) {
|
|
117
|
+
if (section.text.length <= maxChars * 2) {
|
|
118
|
+
out.push(section);
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
const sectionLines = section.text.split('\n');
|
|
122
|
+
let startLine = section.startLine;
|
|
123
|
+
let buf = [];
|
|
124
|
+
let bufChars = 0;
|
|
125
|
+
for (const line of sectionLines) {
|
|
126
|
+
if (bufChars === 0 && line.length > maxChars) {
|
|
127
|
+
let cursor = 0;
|
|
128
|
+
while (cursor < line.length) {
|
|
129
|
+
const piece = line.slice(cursor, cursor + maxChars);
|
|
130
|
+
out.push({
|
|
131
|
+
text: piece,
|
|
132
|
+
startLine,
|
|
133
|
+
endLine: startLine,
|
|
134
|
+
symbol: section.symbol,
|
|
135
|
+
});
|
|
136
|
+
cursor += maxChars;
|
|
137
|
+
}
|
|
138
|
+
startLine += 1;
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
if (bufChars > 0 && bufChars + line.length + 1 > maxChars) {
|
|
142
|
+
out.push({
|
|
143
|
+
text: buf.join('\n'),
|
|
144
|
+
startLine,
|
|
145
|
+
endLine: startLine + buf.length - 1,
|
|
146
|
+
symbol: section.symbol,
|
|
147
|
+
});
|
|
148
|
+
startLine += buf.length;
|
|
149
|
+
buf = [];
|
|
150
|
+
bufChars = 0;
|
|
151
|
+
}
|
|
152
|
+
buf.push(line);
|
|
153
|
+
bufChars += line.length + 1;
|
|
154
|
+
}
|
|
155
|
+
if (buf.length > 0) {
|
|
156
|
+
out.push({
|
|
157
|
+
text: buf.join('\n'),
|
|
158
|
+
startLine,
|
|
159
|
+
endLine: startLine + buf.length - 1,
|
|
160
|
+
symbol: section.symbol,
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
return out;
|
|
165
|
+
}
|
|
166
|
+
function chunkTextByParagraphs(text, maxChars) {
|
|
167
|
+
const trimmed = text.trim();
|
|
168
|
+
if (trimmed.length === 0)
|
|
169
|
+
return [];
|
|
170
|
+
const paragraphs = trimmed
|
|
171
|
+
.split(/\n\s*\n+/)
|
|
172
|
+
.map(p => p.trim())
|
|
173
|
+
.filter(p => p.length > 0);
|
|
174
|
+
const out = [];
|
|
175
|
+
let cursor = 0;
|
|
176
|
+
let buf = '';
|
|
177
|
+
let bufStart = 0;
|
|
178
|
+
for (const p of paragraphs) {
|
|
179
|
+
if (buf.length === 0) {
|
|
180
|
+
buf = p;
|
|
181
|
+
bufStart = cursor;
|
|
182
|
+
}
|
|
183
|
+
else if (buf.length + 2 + p.length <= maxChars) {
|
|
184
|
+
buf += '\n\n' + p;
|
|
185
|
+
}
|
|
186
|
+
else {
|
|
187
|
+
out.push({ text: buf, offset: bufStart });
|
|
188
|
+
buf = p;
|
|
189
|
+
bufStart = cursor;
|
|
190
|
+
}
|
|
191
|
+
cursor += p.length + 2;
|
|
192
|
+
}
|
|
193
|
+
if (buf.length > 0)
|
|
194
|
+
out.push({ text: buf, offset: bufStart });
|
|
195
|
+
return out;
|
|
196
|
+
}
|
|
197
|
+
//# sourceMappingURL=baseIngester.js.map
|