@gitsense/gscb-git-tools 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/dist/cjs/common/logger.js +97 -0
  2. package/dist/cjs/common/logger.js.map +1 -0
  3. package/dist/cjs/common/metadata-manager.js +196 -0
  4. package/dist/cjs/common/metadata-manager.js.map +1 -0
  5. package/dist/cjs/importer/chat-manager/chat-creator.js +90 -0
  6. package/dist/cjs/importer/chat-manager/chat-creator.js.map +1 -0
  7. package/dist/cjs/importer/chat-manager/chat-manager.js +179 -0
  8. package/dist/cjs/importer/chat-manager/chat-manager.js.map +1 -0
  9. package/dist/cjs/importer/chat-manager/constants.js +66 -0
  10. package/dist/cjs/importer/chat-manager/constants.js.map +1 -0
  11. package/dist/cjs/importer/chat-manager/group-manager.js +56 -0
  12. package/dist/cjs/importer/chat-manager/group-manager.js.map +1 -0
  13. package/dist/cjs/importer/chat-manager/hierarchy-manager.js +115 -0
  14. package/dist/cjs/importer/chat-manager/hierarchy-manager.js.map +1 -0
  15. package/dist/cjs/importer/chat-manager/message-manager.js +126 -0
  16. package/dist/cjs/importer/chat-manager/message-manager.js.map +1 -0
  17. package/dist/cjs/importer/constants.js +30 -0
  18. package/dist/cjs/importer/constants.js.map +1 -0
  19. package/dist/cjs/importer/core/directory-processor.js +343 -0
  20. package/dist/cjs/importer/core/directory-processor.js.map +1 -0
  21. package/dist/cjs/importer/core/error-handler.js +64 -0
  22. package/dist/cjs/importer/core/error-handler.js.map +1 -0
  23. package/dist/cjs/importer/core/import-processor.js +167 -0
  24. package/dist/cjs/importer/core/import-processor.js.map +1 -0
  25. package/dist/cjs/importer/core/importer.js +135 -0
  26. package/dist/cjs/importer/core/importer.js.map +1 -0
  27. package/dist/cjs/importer/core/utils.js +66 -0
  28. package/dist/cjs/importer/core/utils.js.map +1 -0
  29. package/dist/cjs/importer/file-processor.js +199 -0
  30. package/dist/cjs/importer/file-processor.js.map +1 -0
  31. package/dist/cjs/importer/language-utils.js +286 -0
  32. package/dist/cjs/importer/language-utils.js.map +1 -0
  33. package/dist/cjs/importer/tree-processor.js +119 -0
  34. package/dist/cjs/importer/tree-processor.js.map +1 -0
  35. package/dist/cjs/index.js +12 -0
  36. package/dist/cjs/index.js.map +1 -0
  37. package/dist/cjs/package.json +3 -0
  38. package/dist/esm/common/logger.js +95 -0
  39. package/dist/esm/common/logger.js.map +1 -0
  40. package/dist/esm/common/metadata-manager.js +194 -0
  41. package/dist/esm/common/metadata-manager.js.map +1 -0
  42. package/dist/esm/importer/chat-manager/chat-creator.js +88 -0
  43. package/dist/esm/importer/chat-manager/chat-creator.js.map +1 -0
  44. package/dist/esm/importer/chat-manager/chat-manager.js +177 -0
  45. package/dist/esm/importer/chat-manager/chat-manager.js.map +1 -0
  46. package/dist/esm/importer/chat-manager/constants.js +48 -0
  47. package/dist/esm/importer/chat-manager/constants.js.map +1 -0
  48. package/dist/esm/importer/chat-manager/group-manager.js +54 -0
  49. package/dist/esm/importer/chat-manager/group-manager.js.map +1 -0
  50. package/dist/esm/importer/chat-manager/hierarchy-manager.js +113 -0
  51. package/dist/esm/importer/chat-manager/hierarchy-manager.js.map +1 -0
  52. package/dist/esm/importer/chat-manager/message-manager.js +124 -0
  53. package/dist/esm/importer/chat-manager/message-manager.js.map +1 -0
  54. package/dist/esm/importer/constants.js +28 -0
  55. package/dist/esm/importer/constants.js.map +1 -0
  56. package/dist/esm/importer/core/directory-processor.js +337 -0
  57. package/dist/esm/importer/core/directory-processor.js.map +1 -0
  58. package/dist/esm/importer/core/error-handler.js +62 -0
  59. package/dist/esm/importer/core/error-handler.js.map +1 -0
  60. package/dist/esm/importer/core/import-processor.js +165 -0
  61. package/dist/esm/importer/core/import-processor.js.map +1 -0
  62. package/dist/esm/importer/core/importer.js +133 -0
  63. package/dist/esm/importer/core/importer.js.map +1 -0
  64. package/dist/esm/importer/core/utils.js +64 -0
  65. package/dist/esm/importer/core/utils.js.map +1 -0
  66. package/dist/esm/importer/file-processor.js +193 -0
  67. package/dist/esm/importer/file-processor.js.map +1 -0
  68. package/dist/esm/importer/language-utils.js +279 -0
  69. package/dist/esm/importer/language-utils.js.map +1 -0
  70. package/dist/esm/importer/tree-processor.js +113 -0
  71. package/dist/esm/importer/tree-processor.js.map +1 -0
  72. package/dist/esm/index.js +4 -0
  73. package/dist/esm/index.js.map +1 -0
  74. package/dist/index.d.ts +215 -0
  75. package/package.json +2 -2
@@ -0,0 +1,62 @@
1
+ import { DatabaseError, ValidationError, NotFoundError } from '@gitsense/gscb-db';
2
+
3
+ /**
4
+ * Component: Error Handler
5
+ * Block-UUID: 237b8cc5-5d1b-4285-82c4-0e1f49459f2c
6
+ * Parent-UUID: 31839e44-58af-43ea-baab-96674d745ad0
7
+ * Version: 6.6.0
8
+ * Purpose: Centralized error handling for GitImporter
9
+ * Language: TypeScript
10
+ * Created-at: 2025-03-15T05:59:20.178Z
11
+ * Authors: Claude 3.5 Sonnet (v5.0.1), Bard, Claude 3.5 Sonnet (v6.0.0), Claude 3.7 Sonnet (v6.3.0), OpenRouter - Gemini Flash 2.0 (v6.4.0), OpenRouter - Gemini Flash 2.0 (v6.5.0), together.ai - DeepSeek v3 (v6.6.0)
12
+ */
13
+ class ErrorHandler {
14
+ logger;
15
+ constructor(logger) {
16
+ this.logger = logger;
17
+ }
18
+ /**
19
+ * Handles errors during the import process.
20
+ * @param error The error to handle.
21
+ * @throws {Error} Always throws an error after handling.
22
+ */
23
+ async handleError(error) {
24
+ let message;
25
+ let isExpected = false;
26
+ if (error instanceof DatabaseError) {
27
+ message = `Database error: ${error.message}`;
28
+ isExpected = true;
29
+ }
30
+ else if (error instanceof ValidationError) {
31
+ message = `Validation error: ${error.message}`;
32
+ isExpected = true;
33
+ }
34
+ else if (error instanceof NotFoundError) {
35
+ message = `Not found: ${error.message}`;
36
+ isExpected = true;
37
+ }
38
+ else if (this.isGitError(error)) {
39
+ message = `Git error: ${error.message}`;
40
+ isExpected = true;
41
+ }
42
+ else if (error instanceof Error) {
43
+ message = `Unexpected error: ${error.message}`;
44
+ }
45
+ else {
46
+ message = `Unknown error: ${String(error)}`;
47
+ }
48
+ this.logger.log(message, error, !isExpected);
49
+ throw error;
50
+ }
51
+ /**
52
+ * Checks if an error is a Git error.
53
+ */
54
+ isGitError(error) {
55
+ return (error instanceof Error &&
56
+ 'code' in error &&
57
+ typeof error.code === 'string');
58
+ }
59
+ }
60
+
61
+ export { ErrorHandler };
62
+ //# sourceMappingURL=error-handler.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"error-handler.js","sources":["../../../../src/importer/core/error-handler.ts"],"sourcesContent":[null],"names":[],"mappings":";;AAAA;;;;;;;;;AASG;MAOU,YAAY,CAAA;AACQ,IAAA,MAAA;AAA7B,IAAA,WAAA,CAA6B,MAAc,EAAA;QAAd,IAAM,CAAA,MAAA,GAAN,MAAM;;AAEnC;;;;AAIG;IACH,MAAM,WAAW,CAAC,KAAc,EAAA;AAC5B,QAAA,IAAI,OAAe;QACnB,IAAI,UAAU,GAAG,KAAK;AAEtB,QAAA,IAAI,KAAK,YAAY,aAAa,EAAE;AAChC,YAAA,OAAO,GAAG,CAAmB,gBAAA,EAAA,KAAK,CAAC,OAAO,EAAE;YAC5C,UAAU,GAAG,IAAI;;AACd,aAAA,IAAI,KAAK,YAAY,eAAe,EAAE;AACzC,YAAA,OAAO,GAAG,CAAqB,kBAAA,EAAA,KAAK,CAAC,OAAO,EAAE;YAC9C,UAAU,GAAG,IAAI;;AACd,aAAA,IAAI,KAAK,YAAY,aAAa,EAAE;AACvC,YAAA,OAAO,GAAG,CAAc,WAAA,EAAA,KAAK,CAAC,OAAO,EAAE;YACvC,UAAU,GAAG,IAAI;;AACd,aAAA,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE;AAC/B,YAAA,OAAO,GAAG,CAAe,WAAA,EAAA,KAA6B,CAAC,OAAO,EAAE;YAChE,UAAU,GAAG,IAAI;;AACd,aAAA,IAAI,KAAK,YAAY,KAAK,EAAE;AAC/B,YAAA,OAAO,GAAG,CAAqB,kBAAA,EAAA,KAAK,CAAC,OAAO,EAAE;;aAC3C;AACH,YAAA,OAAO,GAAG,CAAkB,eAAA,EAAA,MAAM,CAAC,KAAK,CAAC,EAAE;;AAG/C,QAAA,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,UAAU,CAAC;AAC5C,QAAA,MAAM,KAAK;;AAGf;;AAEG;AACK,IAAA,UAAU,CAAC,KAAc,EAAA;QAC7B,QACI,KAAK,YAAY,KAAK;AACtB,YAAA,MAAM,IAAI,KAAK;AACf,YAAA,OAAQ,KAAkB,CAAC,IAAI,KAAK,QAAQ;;AAGvD;;;;"}
@@ -0,0 +1,165 @@
1
+ import { ChatManager } from '../chat-manager/chat-manager.js';
2
+ import { MessageManager } from '../chat-manager/message-manager.js';
3
+ import { FileProcessor } from '../file-processor.js';
4
+ import { TreeProcessor } from '../tree-processor.js';
5
+ import { DirectoryProcessor } from './directory-processor.js';
6
+ import { MetadataManager } from '../../common/metadata-manager.js';
7
+
8
+ /**
9
+ * Component: Import Processor
10
+ * Block-UUID: 5cc5dcf8-63d7-415a-aa21-ffa85e3b8dd9
11
+ * Parent-UUID: 237b8cc5-5d1b-4285-82c4-0e1f49459f2c
12
+ * Version: 6.8.0
13
+ * Purpose: Handles the import process logic for GitImporter with optimized metadata updates
14
+ * Language: TypeScript
15
+ * Created-at: 2025-03-17T04:14:43.902Z
16
+ * Authors: Claude 3.5 Sonnet (v5.0.1), Bard, Claude 3.5 Sonnet (v6.0.0), Claude 3.7 Sonnet (v6.3.0),
17
+ * OpenRouter - Gemini Flash 2.0 (v6.4.0), OpenRouter - Gemini Flash 2.0 (v6.5.0),
18
+ * together.ai - DeepSeek v3 (v6.6.0), Claude 3.7 Sonnet (v6.7.0), together.ai - DeepSeek v3 (v6.8.0)
19
+ */
20
+ class ImportProcessor {
21
+ dbClient;
22
+ gitRepo;
23
+ repoOwner;
24
+ repoName;
25
+ ref;
26
+ refType;
27
+ options;
28
+ logger;
29
+ chatManager;
30
+ messageManager;
31
+ _metadataManager = null;
32
+ _fileProcessor = null;
33
+ _treeProcessor = null;
34
+ constructor(dbClient, gitRepo, repoOwner, repoName, ref, refType, options, logger) {
35
+ this.dbClient = dbClient;
36
+ this.gitRepo = gitRepo;
37
+ this.repoOwner = repoOwner;
38
+ this.repoName = repoName;
39
+ this.ref = ref;
40
+ this.refType = refType;
41
+ this.options = options;
42
+ this.logger = logger;
43
+ this.chatManager = new ChatManager(dbClient, gitRepo, repoOwner, repoName, ref, refType, options, logger);
44
+ this.messageManager = new MessageManager(dbClient, logger);
45
+ }
46
+ /**
47
+ * Sets up the import process with optimized metadata updates
48
+ */
49
+ async setupImport() {
50
+ try {
51
+ // Setup chat hierarchy
52
+ const { refChatId, repoGroupId } = await this.chatManager.setupChatHierarchy();
53
+ // Process import
54
+ const currentCommit = await this.gitRepo.resolveRef(this.ref);
55
+ const refChat = await this.chatManager.getChatById(refChatId);
56
+ const meta = refChat?.meta || {};
57
+ if (meta && meta.status === 'imported' && meta.commit) {
58
+ await this.processExistingImport(meta, currentCommit, refChatId, repoGroupId);
59
+ }
60
+ else {
61
+ await this.processNewImport(refChatId, repoGroupId);
62
+ }
63
+ return { refChatId, repoGroupId };
64
+ }
65
+ catch (error) {
66
+ throw error;
67
+ }
68
+ }
69
+ /**
70
+ * Processes an existing import by checking for changes and performing an incremental import if necessary.
71
+ */
72
+ async processExistingImport(meta, currentCommit, refChatId, repoGroupId) {
73
+ if (!meta.commit)
74
+ throw new Error("No ref commit meta or hash found");
75
+ const metadataManager = this.getMetadataManager();
76
+ if (meta.commit.hash === currentCommit) {
77
+ this.logger.log('Repository already imported with the same commit hash, skipping import');
78
+ return;
79
+ }
80
+ try {
81
+ this.logger.log(`Found previous import with commit hash: ${meta.commit.hash}`);
82
+ // Mark the start of the import
83
+ await metadataManager.markImportStarted(refChatId, this.ref);
84
+ // Get the list of changed entries with their change types
85
+ const changedEntries = await this.gitRepo.getChangedEntriesBetweenCommits(meta.commit?.hash || '', currentCommit, { includeFiles: true, includeDirectories: true });
86
+ if (changedEntries.length > 0) {
87
+ this.logger.log(`Found ${changedEntries.length} changed entries to process`);
88
+ // Process the changed entries
89
+ const directoryProcessor = new DirectoryProcessor(this.dbClient, this.gitRepo, this.getFileProcessor(repoGroupId), this.getTreeProcessor(repoGroupId), this.getMetadataManager(), this.chatManager, this.logger);
90
+ await directoryProcessor.processChangedEntries(changedEntries, refChatId, repoGroupId);
91
+ this.logger.log('Incremental import completed successfully');
92
+ }
93
+ else {
94
+ this.logger.log('No files changed, updating commit hash only');
95
+ }
96
+ await metadataManager.markImportFinished(refChatId);
97
+ }
98
+ catch (error) {
99
+ console.trace();
100
+ this.logger.log('Incremental import failed: ', error instanceof Error ? error.message : String(error));
101
+ await metadataManager.markImportFailed(refChatId, error);
102
+ }
103
+ }
104
+ /**
105
+ * Processes a new import by performing a full import of the repository.
106
+ */
107
+ async processNewImport(refChatId, repoGroupId) {
108
+ const metadataManager = this.getMetadataManager();
109
+ this.logger.log('Performing full import');
110
+ await metadataManager.markImportStarted(refChatId, this.ref);
111
+ // Process all files
112
+ const directoryProcessor = new DirectoryProcessor(this.dbClient, this.gitRepo, this.getFileProcessor(repoGroupId), this.getTreeProcessor(repoGroupId), this.getMetadataManager(), this.chatManager, this.logger);
113
+ await directoryProcessor.processFiles(refChatId);
114
+ await metadataManager.markImportFinished(refChatId);
115
+ this.logger.log('Full import completed successfully');
116
+ }
117
+ /**
118
+ * Gets the chat hierarchy from git-repos down to git-ref
119
+ */
120
+ async getChatHierarchy(refChatId) {
121
+ const hierarchy = [];
122
+ let currentChatId = refChatId;
123
+ while (currentChatId) {
124
+ const chat = await this.chatManager.getChatById(currentChatId);
125
+ if (!chat || !chat.id)
126
+ break;
127
+ hierarchy.unshift({
128
+ id: chat.id,
129
+ type: chat.type
130
+ });
131
+ currentChatId = chat.parentId || 0;
132
+ }
133
+ return hierarchy;
134
+ }
135
+ /**
136
+ * Lazily initializes and returns the MetadataManager instance.
137
+ */
138
+ getMetadataManager() {
139
+ if (!this._metadataManager) {
140
+ this._metadataManager = new MetadataManager(this.dbClient, this.logger, this.gitRepo, this.chatManager);
141
+ }
142
+ return this._metadataManager;
143
+ }
144
+ /**
145
+ * Lazily initializes and returns the FileProcessor instance.
146
+ */
147
+ getFileProcessor(repoGroupId) {
148
+ if (!this._fileProcessor) {
149
+ this._fileProcessor = new FileProcessor(this.dbClient, this.gitRepo, this.chatManager, this.options, repoGroupId, this.logger);
150
+ }
151
+ return this._fileProcessor;
152
+ }
153
+ /**
154
+ * Lazily initializes and returns the TreeProcessor instance.
155
+ */
156
+ getTreeProcessor(repoGroupId) {
157
+ if (!this._treeProcessor) {
158
+ this._treeProcessor = new TreeProcessor(this.gitRepo, this.chatManager, this.getFileProcessor(repoGroupId), repoGroupId, this.logger);
159
+ }
160
+ return this._treeProcessor;
161
+ }
162
+ }
163
+
164
+ export { ImportProcessor };
165
+ //# sourceMappingURL=import-processor.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"import-processor.js","sources":["../../../../src/importer/core/import-processor.ts"],"sourcesContent":[null],"names":[],"mappings":";;;;;;;AAAA;;;;;;;;;;;AAWG;MAeU,eAAe,CAAA;AAQH,IAAA,QAAA;AACA,IAAA,OAAA;AACA,IAAA,SAAA;AACA,IAAA,QAAA;AACA,IAAA,GAAA;AACA,IAAA,OAAA;AACA,IAAA,OAAA;AACA,IAAA,MAAA;AAdJ,IAAA,WAAW;AACX,IAAA,cAAc;IACvB,gBAAgB,GAA2B,IAAI;IAC/C,cAAc,GAAyB,IAAI;IAC3C,cAAc,GAAyB,IAAI;AAEnD,IAAA,WAAA,CACqB,QAAwB,EACxB,OAAmB,EACnB,SAAiB,EACjB,QAAgB,EAChB,GAAW,EACX,OAAgB,EAChB,OAAsB,EACtB,MAAc,EAAA;QAPd,IAAQ,CAAA,QAAA,GAAR,QAAQ;QACR,IAAO,CAAA,OAAA,GAAP,OAAO;QACP,IAAS,CAAA,SAAA,GAAT,SAAS;QACT,IAAQ,CAAA,QAAA,GAAR,QAAQ;QACR,IAAG,CAAA,GAAA,GAAH,GAAG;QACH,IAAO,CAAA,OAAA,GAAP,OAAO;QACP,IAAO,CAAA,OAAA,GAAP,OAAO;QACP,IAAM,CAAA,MAAA,GAAN,MAAM;QAEvB,IAAI,CAAC,WAAW,GAAG,IAAI,WAAW,CAC9B,QAAQ,EACR,OAAO,EACP,SAAS,EACT,QAAQ,EACR,GAAG,EACH,OAAO,EACP,OAAO,EACP,MAAM,CACT;QAED,IAAI,CAAC,cAAc,GAAG,IAAI,cAAc,CAAC,QAAQ,EAAE,MAAM,CAAC;;AAG9D;;AAEG;AACH,IAAA,MAAM,WAAW,GAAA;AACb,QAAA,IAAI;;AAEA,YAAA,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,kBAAkB,EAAE;;AAG9E,YAAA,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC;YAC7D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,SAAS,CAAC;AAC7D,YAAA,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,EAAE;AAEhC,YAAA,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,UAAU,IAAI,IAAI,CAAC,MAAM,EAAE;AACnD,gBAAA,MAAM,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,CAAC;;iBAC1E;gBACH,MAAM,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,WAAW,CAAC;;AAGvD,YAAA,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE;;QACnC,OAAO,KAAK,EAAE;AACZ,YAAA,MAAM,KAAK;;;AAInB;;AAEG;IACK,MAAM,qBAAqB,CAAC,IAAgB,EAAE,aAAqB,EAAE,SAAiB,EAAE,WAAmB,EAAA;QAC/G,IAAI,CAAC,IAAI,CAAC,MAAM;AACZ,YAAA,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC;AAEvD,QAAA,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,EAAE;QAEjD,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,aAAa,EAAE;AACpC,YAAA,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,wEAAwE,CAAC;YACzF;;AAGJ,QAAA,IAAI;AACA,YAAA,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA,wCAAA,EAA2C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAA,CAAE,CAAC;;YAG9E,MAAM,eAAe,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC;;AAG5D,YAAA,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,+BAA+B,CACrE,IAAI,CAAC,MAAM,EAAE,IAAI,IAAI,EAAE,EACvB,aAAa,EACb,EAAE,YAAY,EAAE,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,CACnD;AAED,YAAA,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE;gBAC3B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAS,MAAA,EAAA,cAAc,CAAC,MAAM,CAA6B,2BAAA,CAAA,CAAC;;AAG5E,gBAAA,MAAM,kBAAkB,GAAG,IAAI,kBAAkB,CAC7C,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,EAClC,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,EAClC,IAAI,CAAC,kBAAkB,EAAE,EACzB,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,MAAM,CACd;gBACD,MAAM,kBAAkB,CAAC,qBAAqB,CAAC,cAAc,EAAE,SAAS,EAAE,WAAW,CAAC;AAEtF,gBAAA,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,2CAA2C,CAAC;;iBACzD;AACH,gBAAA,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,6CAA6C,CAAC;;AAElE,YAAA,MAAM,eAAe,CAAC,kBAAkB,CAAC,SAAS,CAAC;;QACrD,OAAM,KAAK,EAAE;YACX,OAAO,CAAC,KAAK,EAAE;YACf,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,6BAA6B,EAAE,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;YACtG,MAAM,eAAe,CAAC,gBAAgB,CAAC,SAAS,EAAE,KAAc,CAAC;;;AAIzE;;AAEG;AACK,IAAA,MAAM,gBAAgB,CAAC,SAAiB,EAAE,WAAmB,EAAA;AACjE,QAAA,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,EAAE;AAEjD,QAAA,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,wBAAwB,CAAC;QACzC,MAAM,eAAe,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC;;AAG5D,QAAA,MAAM,kBAAkB,GAAG,IAAI,kBAAkB,CAC7C,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,EAClC,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,EAClC,IAAI,CAAC,kBAAkB,EAAE,EACzB,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,MAAM,CACd;AACD,QAAA,MAAM,kBAAkB,CAAC,YAAY,CAAC,SAAS,CAAC;AAEhD,QAAA,MAAM,eAAe,CAAC,kBAAkB,CAAC,SAAS,CAAC;AACnD,QAAA,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,oCAAoC,CAAC;;AAGzD;;AAEG;IACK,MAAM,gBAAgB,CAAC,SAAiB,EAAA;QAC5C,MAAM,SAAS,GAAwC,EAAE;QACzD,IAAI,aAAa,GAAG,SAAS;QAE7B,OAAO,aAAa,EAAE;YAClB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,aAAa,CAAC;AAC9D,YAAA,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE;gBAAE;YAEvB,SAAS,CAAC,OAAO,CAAC;gBACd,EAAE,EAAE,IAAI,CAAC,EAAE;gBACX,IAAI,EAAE,IAAI,CAAC;AACd,aAAA,CAAC;AAEF,YAAA,aAAa,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC;;AAGtC,QAAA,OAAO,SAAS;;AAGpB;;AAEG;IACK,kBAAkB,GAAA;AACtB,QAAA,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE;YACxB,IAAI,CAAC,gBAAgB,GAAG,IAAI,eAAe,CACvC,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,WAAW,CACnB;;QAEL,OAAO,IAAI,CAAC,gBAAgB;;AAGhC;;AAEG;AACK,IAAA,gBAAgB,CAAC,WAAmB,EAAA;AACxC,QAAA,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE;AACtB,YAAA,IAAI,CAAC,cAAc,GAAG,IAAI,aAAa,CACnC,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,OAAO,EACZ,WAAW,EACX,IAAI,CAAC,MAAM,CACd;;QAEL,OAAO,IAAI,CAAC,cAAc;;AAG9B;;AAEG;AACK,IAAA,gBAAgB,CAAC,WAAmB,EAAA;AACxC,QAAA,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE;YACtB,IAAI,CAAC,cAAc,GAAG,IAAI,aAAa,CACnC,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,EAClC,WAAW,EACX,IAAI,CAAC,MAAM,CACd;;QAEL,OAAO,IAAI,CAAC,cAAc;;AAEjC;;;;"}
@@ -0,0 +1,133 @@
1
+ import { DatabaseClient } from '@gitsense/gscb-db';
2
+ import { Repository } from '@gitsense/gscb-git';
3
+ import { Logger } from '../../common/logger.js';
4
+ import { DEFAULT_OPTIONS } from '../constants.js';
5
+ import { ImportProcessor } from './import-processor.js';
6
+ import { ErrorHandler } from './error-handler.js';
7
+
8
+ /**
9
+ * Component: Git Repository Importer (Core)
10
+ * Block-UUID: 237b8cc5-5d1b-4285-82c4-0e1f49459f2c
11
+ * Parent-UUID: 31839e44-58af-43ea-baab-96674d745ad0
12
+ * Version: 6.6.0
13
+ * Purpose: Core implementation of GitImporter class
14
+ * Language: TypeScript
15
+ * Created-at: 2025-03-15T05:57:33.811Z
16
+ * Authors: Claude 3.5 Sonnet (v5.0.1), Bard, Claude 3.5 Sonnet (v6.0.0), Claude 3.7 Sonnet (v6.3.0), OpenRouter - Gemini Flash 2.0 (v6.4.0), OpenRouter - Gemini Flash 2.0 (v6.5.0), together.ai - DeepSeek v3 (v6.6.0)
17
+ */
18
+ class GitImporter {
19
+ repoPath;
20
+ repoOwner;
21
+ repoName;
22
+ ref;
23
+ refType;
24
+ dbPath;
25
+ options;
26
+ gitRepo;
27
+ dbClient;
28
+ logger;
29
+ initialized = false;
30
+ importProcessor;
31
+ errorHandler;
32
+ constructor(repoPath, repoOwner, repoName, ref, refType, dbPath, options = {}) {
33
+ this.repoPath = repoPath;
34
+ this.repoOwner = repoOwner;
35
+ this.repoName = repoName;
36
+ this.ref = ref;
37
+ this.refType = refType;
38
+ this.dbPath = dbPath;
39
+ this.options = options;
40
+ // Validate input parameters
41
+ if (!repoPath)
42
+ throw new Error('Repository path is required');
43
+ if (!repoOwner)
44
+ throw new Error('Repository owner is required');
45
+ if (!repoName)
46
+ throw new Error('Repository name is required');
47
+ if (!ref)
48
+ throw new Error('Reference is required');
49
+ if (!['branch', 'commit', 'tag'].includes(refType)) {
50
+ throw new Error('Reference type must be one of: branch, commit, tag');
51
+ }
52
+ if (!dbPath)
53
+ throw new Error('Database path is required');
54
+ // Merge default options
55
+ this.options = {
56
+ ...DEFAULT_OPTIONS,
57
+ ...options
58
+ };
59
+ const gitConfig = {
60
+ repoPath,
61
+ ref,
62
+ };
63
+ this.gitRepo = new Repository(gitConfig);
64
+ const dbConfig = {
65
+ path: dbPath,
66
+ poolSize: 5,
67
+ timeout: 5000,
68
+ };
69
+ this.dbClient = new DatabaseClient(dbConfig);
70
+ this.logger = new Logger(this.options.verbose || false);
71
+ // Initialize modules
72
+ this.importProcessor = new ImportProcessor(this.dbClient, this.gitRepo, this.repoOwner, this.repoName, this.ref, this.refType, this.options, this.logger);
73
+ this.errorHandler = new ErrorHandler(this.logger);
74
+ }
75
+ /**
76
+ * Initializes the Git repository, ref commit, and database connection.
77
+ */
78
+ async initialize() {
79
+ if (this.initialized) {
80
+ return;
81
+ }
82
+ this.logger.log('Initializing...');
83
+ try {
84
+ await this.gitRepo.initialize();
85
+ this.logger.log('Git repository initialized');
86
+ await this.dbClient.initialize();
87
+ this.logger.log('Database initialized');
88
+ this.initialized = true;
89
+ }
90
+ catch (error) {
91
+ throw new Error(`Initialization failed: ${error instanceof Error ? error.message : String(error)}`);
92
+ }
93
+ }
94
+ /**
95
+ * Imports the repository by setting up the chat hierarchy.
96
+ *
97
+ * @returns An object containing the ID of the git-ref chat
98
+ * @throws {Error} If initialization fails or chat hierarchy setup fails
99
+ */
100
+ async import() {
101
+ try {
102
+ await this.initialize();
103
+ return await this.importProcessor.setupImport();
104
+ }
105
+ catch (error) {
106
+ await this.errorHandler.handleError(error);
107
+ // This line is needed to satisfy TypeScript, though it will never be reached
108
+ // because handleError always throws
109
+ return { refChatId: -1 };
110
+ }
111
+ finally {
112
+ await this.cleanup();
113
+ }
114
+ }
115
+ /**
116
+ * Cleans up resources by closing the database connection.
117
+ */
118
+ async cleanup() {
119
+ if (this.dbClient.isInitialized()) {
120
+ try {
121
+ await this.dbClient.close();
122
+ this.logger.log('Database connection closed');
123
+ }
124
+ catch (error) {
125
+ this.logger.log('Error during database cleanup:', error, true);
126
+ }
127
+ }
128
+ this.initialized = false;
129
+ }
130
+ }
131
+
132
+ export { GitImporter };
133
+ //# sourceMappingURL=importer.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"importer.js","sources":["../../../../src/importer/core/importer.ts"],"sourcesContent":[null],"names":[],"mappings":";;;;;;;AAAA;;;;;;;;;AASG;MAYU,WAAW,CAAA;AASC,IAAA,QAAA;AACA,IAAA,SAAA;AACA,IAAA,QAAA;AACA,IAAA,GAAA;AACA,IAAA,OAAA;AACA,IAAA,MAAA;AACA,IAAA,OAAA;AAdJ,IAAA,OAAO;AACP,IAAA,QAAQ;AACR,IAAA,MAAM;IACf,WAAW,GAAY,KAAK;AAC5B,IAAA,eAAe;AACf,IAAA,YAAY;AAEpB,IAAA,WAAA,CACqB,QAAgB,EAChB,SAAiB,EACjB,QAAgB,EAChB,GAAW,EACX,OAAgB,EAChB,MAAc,EACd,UAAyB,EAAE,EAAA;QAN3B,IAAQ,CAAA,QAAA,GAAR,QAAQ;QACR,IAAS,CAAA,SAAA,GAAT,SAAS;QACT,IAAQ,CAAA,QAAA,GAAR,QAAQ;QACR,IAAG,CAAA,GAAA,GAAH,GAAG;QACH,IAAO,CAAA,OAAA,GAAP,OAAO;QACP,IAAM,CAAA,MAAA,GAAN,MAAM;QACN,IAAO,CAAA,OAAA,GAAP,OAAO;;AAGxB,QAAA,IAAI,CAAC,QAAQ;AAAE,YAAA,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC;AAC7D,QAAA,IAAI,CAAC,SAAS;AAAE,YAAA,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC;AAC/D,QAAA,IAAI,CAAC,QAAQ;AAAE,YAAA,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC;AAC7D,QAAA,IAAI,CAAC,GAAG;AAAE,YAAA,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC;AAClD,QAAA,IAAI,CAAC,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE;AAChD,YAAA,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC;;AAEzE,QAAA,IAAI,CAAC,MAAM;AAAE,YAAA,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC;;QAGzD,IAAI,CAAC,OAAO,GAAG;AACX,YAAA,GAAG,eAAe;AAClB,YAAA,GAAG;SACN;AAED,QAAA,MAAM,SAAS,GAAc;YACzB,QAAQ;YACR,GAAG;SACN;QAED,IAAI,CAAC,OAAO,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC;AAExC,QAAA,MAAM,QAAQ,GAAmB;AAC7B,YAAA,IAAI,EAAE,MAAM;AACZ,YAAA,QAAQ,EAAE,CAAC;AACX,YAAA,OAAO,EAAE,IAAI;SAChB;QAED,IAAI,CAAC,QAAQ,GAAG,IAAI,cAAc,CAAC,QAAQ,CAAC;AAC5C,QAAA,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC;;AAGvD,QAAA,IAAI,CAAC,eAAe,GAAG,IAAI,eAAe,CACtC,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,GAAG,EACR,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,MAAM,CACd;QAED,IAAI,CAAC,YAAY,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC;;AAGrD;;AAEG;AACH,IAAA,MAAM,UAAU,GAAA;AACZ,QAAA,IAAI,IAAI,CAAC,WAAW,EAAE;YAClB;;AAGJ,QAAA,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,iBAAiB,CAAC;AAElC,QAAA,IAAI;AACA,YAAA,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE;AAC/B,YAAA,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,4BAA4B,CAAC;AAE7C,YAAA,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE;AAChC,YAAA,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,sBAAsB,CAAC;AAEvC,YAAA,IAAI,CAAC,WAAW,GAAG,IAAI;;QACzB,OAAO,KAAK,EAAE;YACZ,MAAM,IAAI,KAAK,CAAC,CAAA,uBAAA,EAA0B,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAE,CAAA,CAAC;;;AAI3G;;;;;AAKG;AACH,IAAA,MAAM,MAAM,GAAA;AACR,QAAA,IAAI;AACA,YAAA,MAAM,IAAI,CAAC,UAAU,EAAE;AACvB,YAAA,OAAO,MAAM,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE;;QACjD,OAAO,KAAK,EAAE;YACZ,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,KAAK,CAAC;;;AAG1C,YAAA,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE;;gBAClB;AACN,YAAA,MAAM,IAAI,CAAC,OAAO,EAAE;;;AAI5B;;AAEG;AACK,IAAA,MAAM,OAAO,GAAA;AACjB,QAAA,IAAI,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,EAAE;AAC/B,YAAA,IAAI;AACA,gBAAA,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE;AAC3B,gBAAA,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,4BAA4B,CAAC;;YAC/C,OAAO,KAAK,EAAE;gBACZ,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,gCAAgC,EAAE,KAAK,EAAE,IAAI,CAAC;;;AAGtE,QAAA,IAAI,CAAC,WAAW,GAAG,KAAK;;AAE/B;;;;"}
@@ -0,0 +1,64 @@
1
+ /**
2
+ * Component: Utilities
3
+ * Block-UUID: 237b8cc5-5d1b-4285-82c4-0e1f49459f2c
4
+ * Parent-UUID: 31839e44-58af-43ea-baab-96674d745ad0
5
+ * Version: 6.6.0
6
+ * Purpose: Utility functions for GitImporter
7
+ * Language: TypeScript
8
+ * Created-at: 2025-03-15T06:02:35.714Z
9
+ * Authors: Claude 3.5 Sonnet (v5.0.1), Bard, Claude 3.5 Sonnet (v6.0.0), Claude 3.7 Sonnet (v6.3.0), OpenRouter - Gemini Flash 2.0 (v6.4.0), OpenRouter - Gemini Flash 2.0 (v6.5.0), together.ai - DeepSeek v3 (v6.6.0)
10
+ */
11
+ class Utils {
12
+ gitRepo;
13
+ logger;
14
+ constructor(gitRepo, logger) {
15
+ this.gitRepo = gitRepo;
16
+ this.logger = logger;
17
+ }
18
+ /**
19
+ * Normalizes a file path.
20
+ */
21
+ normalizePath(path) {
22
+ return path.replace(/\\/g, '/').replace(/\/+/g, '/').replace(/\/$/, '');
23
+ }
24
+ /**
25
+ * Validates that the repository exists and the reference is valid.
26
+ * @throws {Error} If the repository doesn't exist or the reference is invalid
27
+ */
28
+ async validateRepository(repoPath, ref, refType) {
29
+ if (!this.logger)
30
+ throw new Error('Logger not defined. Unable to call validateRepository');
31
+ if (!this.gitRepo)
32
+ throw new Error('Logger not defined. Unable to call validateRepository');
33
+ try {
34
+ this.logger.log("Validating repository...");
35
+ const repoInfo = await this.gitRepo.getInfo();
36
+ if (!repoInfo) {
37
+ throw new Error(`Repository not found at path: ${repoPath}`);
38
+ }
39
+ // Validate that the reference exists
40
+ const refExists = await this.gitRepo.hasRef(ref);
41
+ if (!refExists) {
42
+ throw new Error(`Reference ${ref} of type ${refType} not found in repository`);
43
+ }
44
+ this.logger.log("Repository validation successful");
45
+ }
46
+ catch (error) {
47
+ throw new Error(`Repository validation failed: ${error instanceof Error ? error.message : String(error)}`);
48
+ }
49
+ }
50
+ /**
51
+ * Determines the latest commit from an array of GitCommitInfo objects.
52
+ */
53
+ static getLatestCommit(commits) {
54
+ if (commits.length === 0) {
55
+ // If no commits, return an empty GitCommitInfo
56
+ return null;
57
+ }
58
+ // Find the commit with the latest timestamp
59
+ return commits.reduce((latest, commit) => commit.timestamp > latest.timestamp ? commit : latest);
60
+ }
61
+ }
62
+
63
+ export { Utils };
64
+ //# sourceMappingURL=utils.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"utils.js","sources":["../../../../src/importer/core/utils.ts"],"sourcesContent":[null],"names":[],"mappings":"AAAA;;;;;;;;;AASG;MAMU,KAAK,CAAA;AAEO,IAAA,OAAA;AACA,IAAA,MAAA;IAFrB,WACqB,CAAA,OAAoB,EACpB,MAAe,EAAA;QADf,IAAO,CAAA,OAAA,GAAP,OAAO;QACP,IAAM,CAAA,MAAA,GAAN,MAAM;;AAG3B;;AAEG;AACH,IAAA,aAAa,CAAC,IAAY,EAAA;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;;AAG3E;;;AAGG;AACH,IAAA,MAAM,kBAAkB,CAAC,QAAgB,EAAE,GAAW,EAAE,OAAoC,EAAA;QACxF,IAAI,CAAC,IAAI,CAAC,MAAM;AAAE,YAAA,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC;QAC1F,IAAI,CAAC,IAAI,CAAC,OAAO;AAAE,YAAA,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC;AAE3F,QAAA,IAAI;AACA,YAAA,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,0BAA0B,CAAC;YAE3C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE;YAC7C,IAAI,CAAC,QAAQ,EAAE;AACX,gBAAA,MAAM,IAAI,KAAK,CAAC,iCAAiC,QAAQ,CAAA,CAAE,CAAC;;;YAIhE,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC;YAChD,IAAI,CAAC,SAAS,EAAE;gBACZ,MAAM,IAAI,KAAK,CAAC,CAAA,UAAA,EAAa,GAAG,CAAY,SAAA,EAAA,OAAO,CAA0B,wBAAA,CAAA,CAAC;;AAGlF,YAAA,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,kCAAkC,CAAC;;QACrD,OAAO,KAAK,EAAE;YACZ,MAAM,IAAI,KAAK,CAAC,CAAA,8BAAA,EAAiC,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAE,CAAA,CAAC;;;AAIlH;;AAEG;IACH,OAAO,eAAe,CAAC,OAAwB,EAAA;AAC3C,QAAA,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;;AAEtB,YAAA,OAAO,IAAI;;;QAIf,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,MAAM,KACjC,MAAM,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,GAAG,MAAM,GAAG,MAAM,CACxD;;AAER;;;;"}
@@ -0,0 +1,193 @@
1
+ import path from 'path';
2
+ import { FSUtils } from '@gitsense/gscb-utils';
3
+ import { getHighlightSyntax, detectLanguage } from './language-utils.js';
4
+
5
+ /**
6
+ * Component: File Processor
7
+ * Block-UUID: 27a7202c-94f0-4b35-8a34-c580a5f346a5
8
+ * Parent-UUID: 73378071-3284-4b9d-9354-436e49ed2f75
9
+ * Version: 5.1.0
10
+ * Description: Processes Git repository files for import, handling file creation, updates, and filtering
11
+ * Language: TypeScript
12
+ * Created-at: 2025-02-28T14:45:00.000Z
13
+ * Updated-at: 2025-03-24T00:45:00.000Z
14
+ * Authors: Claude 3.5 Sonnet (v3.0.0), Bard, together.ai - DeepSeek v3 (v3.3.0), together.ai - DeepSeek v3 (v4.0.0), together.ai - DeepSeek v3 (v5.0.0), Claude 3.7 Sonnet (v5.1.0)
15
+ */
16
+ class FileProcessor {
17
+ dbClient;
18
+ gitRepo;
19
+ chatManager;
20
+ options;
21
+ repoGroupId;
22
+ logger;
23
+ constructor(dbClient, gitRepo, chatManager, options, repoGroupId, logger) {
24
+ this.dbClient = dbClient;
25
+ this.gitRepo = gitRepo;
26
+ this.chatManager = chatManager;
27
+ this.options = options;
28
+ this.repoGroupId = repoGroupId;
29
+ this.logger = logger;
30
+ }
31
+ /**
32
+ * Estimates tokens using 1.33 tokens per word.
33
+ */
34
+ estimateTokens(text) {
35
+ const wordCount = text.split(/\s+/).filter(Boolean).length;
36
+ return Math.ceil(wordCount * 1.33);
37
+ }
38
+ /**
39
+ * Processes a file and returns its GitCommitInfo.
40
+ * If the file already exists in the database, it will update the existing chat.
41
+ */
42
+ async processFile(filePath, parentChatId, orderWeight) {
43
+ try {
44
+ // Check if a chat for this file already exists
45
+ const fileName = path.basename(filePath);
46
+ const childChats = await this.dbClient.getChatsByParentId(parentChatId);
47
+ const existingFileChat = childChats.find(chat => chat.type === 'git-blob' &&
48
+ chat.name === fileName &&
49
+ chat.meta?.path === filePath);
50
+ if (existingFileChat) {
51
+ // Update existing file chat
52
+ return await this.updateExistingFile(existingFileChat.id, filePath);
53
+ }
54
+ else {
55
+ // Create new file chat
56
+ return await this.createNewFile(filePath, parentChatId, orderWeight);
57
+ }
58
+ }
59
+ catch (error) {
60
+ throw new Error(`Failed to process file '${filePath}': ${error instanceof Error ? error.message : String(error)}`);
61
+ }
62
+ }
63
+ /**
64
+ * Creates a new file chat.
65
+ */
66
+ async createNewFile(filePath, parentChatId, orderWeight) {
67
+ const fileInfo = await this.gitRepo.getFileInfo(filePath);
68
+ const fileCommitHash = await this.gitRepo.getFileCommitHash(filePath);
69
+ const commitInfo = await this.gitRepo.getCommitInfo(fileCommitHash);
70
+ const content = await this.gitRepo.readFile(filePath);
71
+ const now = new Date().toISOString();
72
+ const type = 'git-blob';
73
+ // Estimate tokens for content and metadata
74
+ const contentTokens = this.estimateTokens(content.toString());
75
+ const metaTokens = this.estimateTokens(JSON.stringify({
76
+ path: filePath,
77
+ name: path.basename(filePath),
78
+ commit: commitInfo,
79
+ blob: fileInfo.hash,
80
+ size: fileInfo.size,
81
+ language: await detectLanguage(filePath),
82
+ highlight: await getHighlightSyntax(filePath)
83
+ }));
84
+ const meta = {
85
+ type,
86
+ path: filePath,
87
+ name: path.basename(filePath),
88
+ commit: commitInfo,
89
+ blob: fileInfo.hash,
90
+ size: fileInfo.size,
91
+ language: await detectLanguage(filePath),
92
+ highlight: await getHighlightSyntax(filePath),
93
+ tokens: {
94
+ content: { estimate: contentTokens, estimatedAt: now },
95
+ metadata: { estimate: metaTokens, estimatedAt: now }
96
+ },
97
+ importStartedAt: now,
98
+ importFinishedAt: now,
99
+ status: 'imported'
100
+ };
101
+ await this.chatManager.createChat(type, path.basename(filePath), parentChatId, this.repoGroupId, meta, content.toString(), orderWeight);
102
+ return commitInfo;
103
+ }
104
+ /**
105
+ * Updates an existing file chat.
106
+ */
107
+ async updateExistingFile(chatId, filePath) {
108
+ // Get the current file information
109
+ const fileInfo = await this.gitRepo.getFileInfo(filePath);
110
+ const fileCommitHash = await this.gitRepo.getFileCommitHash(filePath);
111
+ const commitInfo = await this.gitRepo.getCommitInfo(fileCommitHash);
112
+ const content = await this.gitRepo.readFile(filePath);
113
+ const now = new Date().toISOString();
114
+ // Get the existing chat and meta
115
+ const chat = await this.chatManager.getChatById(chatId);
116
+ const oldMeta = chat.meta;
117
+ // Skip update if the commit hash hasn't changed
118
+ if (oldMeta.commit?.hash === commitInfo.hash) {
119
+ this.logger.log(`Skipping update for file '${filePath}' (commit hash unchanged)`);
120
+ return commitInfo;
121
+ }
122
+ // Estimate tokens for content and metadata
123
+ const contentTokens = this.estimateTokens(content.toString());
124
+ const metaTokens = this.estimateTokens(JSON.stringify({
125
+ path: filePath,
126
+ name: path.basename(filePath),
127
+ commit: commitInfo,
128
+ blob: fileInfo.hash,
129
+ size: fileInfo.size,
130
+ language: await detectLanguage(filePath),
131
+ highlight: await getHighlightSyntax(filePath)
132
+ }));
133
+ // Create updated metadata
134
+ const updatedMeta = {
135
+ ...oldMeta,
136
+ commit: commitInfo,
137
+ blob: fileInfo.hash,
138
+ size: fileInfo.size,
139
+ importFinishedAt: now,
140
+ status: 'imported'
141
+ };
142
+ // We don't want to override tokens.analysis with a shallow copy
143
+ updatedMeta.tokens.content = { estimate: contentTokens, estimatedAt: now },
144
+ updatedMeta.tokens.metadata = { estimate: metaTokens, estimatedAt: now };
145
+ // Update the chat metadata
146
+ await this.chatManager.updateFileMetadata(chatId, filePath, commitInfo);
147
+ // Update the meta message
148
+ const metaMessage = await this.chatManager.getMessageByChatIdAndType(chatId, 'chat-meta');
149
+ await this.chatManager.updateMessage(metaMessage.id, {
150
+ message: JSON.stringify(updatedMeta, null, 2)
151
+ });
152
+ // Update the main message with new content
153
+ const mainMessage = await this.chatManager.getMessageByChatIdAndType(chatId, 'git-blob');
154
+ const gitBlobMessage = content.toString();
155
+ await this.chatManager.updateMessage(mainMessage.id, {
156
+ message: gitBlobMessage
157
+ });
158
+ return commitInfo;
159
+ }
160
+ /**
161
+ * Determines whether a file should be skipped based on the provided patterns.
162
+ */
163
+ async shouldSkipFile(filePath) {
164
+ const { skipPatterns, includePatterns, skipBinaryFiles } = this.options;
165
+ // Check against skip patterns first (fastest check)
166
+ if (skipPatterns?.some(pattern => pattern.test(filePath))) {
167
+ this.logger.log(`Skipping file '${filePath}' (matched skip pattern)`);
168
+ return true;
169
+ }
170
+ // Check if file is binary (if skipBinaryFiles is enabled)
171
+ if (skipBinaryFiles) {
172
+ try {
173
+ const isBinary = await FSUtils.isBinaryFile(filePath);
174
+ if (isBinary) {
175
+ this.logger.log(`Skipping binary file '${filePath}'`);
176
+ return true;
177
+ }
178
+ }
179
+ catch (error) {
180
+ this.logger.log(`Error checking if file '${filePath}' is binary: ${error instanceof Error ? error.message : String(error)}`);
181
+ }
182
+ }
183
+ // Check against include patterns if specified
184
+ if (includePatterns?.length && !includePatterns.some(pattern => pattern.test(filePath))) {
185
+ this.logger.log(`Skipping file '${filePath}' (did not match include pattern)`);
186
+ return true;
187
+ }
188
+ return false;
189
+ }
190
+ }
191
+
192
+ export { FileProcessor };
193
+ //# sourceMappingURL=file-processor.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"file-processor.js","sources":["../../../src/importer/file-processor.ts"],"sourcesContent":[null],"names":[],"mappings":";;;;AAAA;;;;;;;;;;AAUG;MAaU,aAAa,CAAA;AAED,IAAA,QAAA;AACA,IAAA,OAAA;AACA,IAAA,WAAA;AACA,IAAA,OAAA;AACA,IAAA,WAAA;AACA,IAAA,MAAA;IANrB,WACqB,CAAA,QAAwB,EACxB,OAAmB,EACnB,WAAwB,EACxB,OAAsB,EACtB,WAAmB,EACnB,MAAc,EAAA;QALd,IAAQ,CAAA,QAAA,GAAR,QAAQ;QACR,IAAO,CAAA,OAAA,GAAP,OAAO;QACP,IAAW,CAAA,WAAA,GAAX,WAAW;QACX,IAAO,CAAA,OAAA,GAAP,OAAO;QACP,IAAW,CAAA,WAAA,GAAX,WAAW;QACX,IAAM,CAAA,MAAA,GAAN,MAAM;;AAG3B;;AAEG;AACK,IAAA,cAAc,CAAC,IAAY,EAAA;AAC/B,QAAA,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM;QAC1D,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;;AAGtC;;;AAGG;AACH,IAAA,MAAM,WAAW,CAAC,QAAgB,EAAE,YAAoB,EAAE,WAAmB,EAAA;AACzE,QAAA,IAAI;;YAEA,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACxC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,YAAY,CAAC;AACvE,YAAA,MAAM,gBAAgB,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,IACzC,IAAI,CAAC,IAAI,KAAK,UAAU;gBACxB,IAAI,CAAC,IAAI,KAAK,QAAQ;AACrB,gBAAA,IAAI,CAAC,IAAY,EAAE,IAAI,KAAK,QAAQ,CACxC;YAED,IAAI,gBAAgB,EAAE;;gBAElB,OAAO,MAAM,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,EAAG,EAAE,QAAQ,CAAC;;iBACjE;;gBAEH,OAAO,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,YAAY,EAAE,WAAW,CAAC;;;QAE1E,OAAO,KAAK,EAAE;YACZ,MAAM,IAAI,KAAK,CAAC,CAA2B,wBAAA,EAAA,QAAQ,MAAM,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAE,CAAA,CAAC;;;AAI1H;;AAEG;AACK,IAAA,MAAM,aAAa,CAAC,QAAgB,EAAE,YAAoB,EAAE,WAAmB,EAAA;QACnF,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC;QACzD,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,QAAQ,CAAC;QACrE,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,cAAc,CAAC;QACnE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACrD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACpC,MAAM,IAAI,GAAG,UAAU;;QAGvB,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;QAC7D,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC;AAClD,YAAA,IAAI,EAAE,QAAQ;AACd,YAAA,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;AAC7B,YAAA,MAAM,EAAE,UAAU;YAClB,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,IAAI,EAAE,QAAQ,CAAC,IAAI;AACnB,YAAA,QAAQ,EAAE,MAAM,cAAc,CAAC,QAAQ,CAAC;AACxC,YAAA,SAAS,EAAE,MAAM,kBAAkB,CAAC,QAAQ;AAC/C,SAAA,CAAC,CAAC;AAEH,QAAA,MAAM,IAAI,GAAgB;YACtB,IAAI;AACJ,YAAA,IAAI,EAAE,QAAQ;AACd,YAAA,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;AAC7B,YAAA,MAAM,EAAE,UAAU;YAClB,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,IAAI,EAAE,QAAQ,CAAC,IAAI;AACnB,YAAA,QAAQ,EAAE,MAAM,cAAc,CAAC,QAAQ,CAAC;AACxC,YAAA,SAAS,EAAE,MAAM,kBAAkB,CAAC,QAAQ,CAAC;AAC7C,YAAA,MAAM,EAAE;gBACJ,OAAO,EAAE,EAAE,QAAQ,EAAE,aAAa,EAAE,WAAW,EAAE,GAAG,EAAE;gBACtD,QAAQ,EAAE,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,GAAG;AACrD,aAAA;AACD,YAAA,eAAe,EAAE,GAAG;AACpB,YAAA,gBAAgB,EAAE,GAAG;AACrB,YAAA,MAAM,EAAE;SACX;AAED,QAAe,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAC5C,IAAI,EACJ,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EACvB,YAAY,EACZ,IAAI,CAAC,WAAW,EAChB,IAAI,EACJ,OAAO,CAAC,QAAQ,EAAE,EAClB,WAAW;AAGf,QAAA,OAAO,UAAU;;AAGrB;;AAEG;AACK,IAAA,MAAM,kBAAkB,CAAC,MAAc,EAAE,QAAgB,EAAA;;QAE7D,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC;QACzD,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,QAAQ,CAAC;QACrE,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,cAAc,CAAC;QACnE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACrD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;;QAGpC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,MAAM,CAAC;AACvD,QAAA,MAAM,OAAO,GAAG,IAAI,CAAC,IAAmB;;QAGxC,IAAI,OAAO,CAAC,MAAM,EAAE,IAAI,KAAK,UAAU,CAAC,IAAI,EAAE;YAC1C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAA6B,0BAAA,EAAA,QAAQ,CAA2B,yBAAA,CAAA,CAAC;AACjF,YAAA,OAAO,UAAU;;;QAIrB,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;QAC7D,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC;AAClD,YAAA,IAAI,EAAE,QAAQ;AACd,YAAA,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;AAC7B,YAAA,MAAM,EAAE,UAAU;YAClB,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,IAAI,EAAE,QAAQ,CAAC,IAAI;AACnB,YAAA,QAAQ,EAAE,MAAM,cAAc,CAAC,QAAQ,CAAC;AACxC,YAAA,SAAS,EAAE,MAAM,kBAAkB,CAAC,QAAQ;AAC/C,SAAA,CAAC,CAAC;;AAGH,QAAA,MAAM,WAAW,GAAgB;AAC7B,YAAA,GAAG,OAAO;AACV,YAAA,MAAM,EAAE,UAAU;YAClB,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,IAAI,EAAE,QAAQ,CAAC,IAAI;AACnB,YAAA,gBAAgB,EAAE,GAAG;AACrB,YAAA,MAAM,EAAE;SACX;;AAGD,QAAA,WAAW,CAAC,MAAM,CAAC,OAAO,GAAG,EAAE,QAAQ,EAAE,aAAa,EAAE,WAAW,EAAE,GAAG,EAAE;AAC1E,YAAA,WAAW,CAAC,MAAM,CAAC,QAAQ,GAAG,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,GAAG,EAAE;;AAGxE,QAAA,MAAM,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC,MAAM,EAAE,QAAQ,EAAE,UAAU,CAAC;;AAGvE,QAAA,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,yBAAyB,CAAC,MAAM,EAAE,WAAW,CAAC;QACzF,MAAM,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,WAAW,CAAC,EAAG,EAAE;YAClD,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;AAC/C,SAAA,CAAC;;AAGF,QAAA,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,yBAAyB,CAAC,MAAM,EAAE,UAAU,CAAC;AACxF,QAAA,MAAM,cAAc,GAAG,OAAO,CAAC,QAAQ,EAAE;QACzC,MAAM,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,WAAW,CAAC,EAAG,EAAE;AAClD,YAAA,OAAO,EAAE;AACZ,SAAA,CAAC;AAEF,QAAA,OAAO,UAAU;;AAGrB;;AAEG;IACH,MAAM,cAAc,CAAC,QAAgB,EAAA;QACjC,MAAM,EAAE,YAAY,EAAE,eAAe,EAAE,eAAe,EAAE,GAAG,IAAI,CAAC,OAAO;;AAGvE,QAAA,IAAI,YAAY,EAAE,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE;YACvD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAkB,eAAA,EAAA,QAAQ,CAA0B,wBAAA,CAAA,CAAC;AACrE,YAAA,OAAO,IAAI;;;QAIf,IAAI,eAAe,EAAE;AACjB,YAAA,IAAI;gBACA,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC;gBAErD,IAAI,QAAQ,EAAE;oBACV,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAyB,sBAAA,EAAA,QAAQ,CAAG,CAAA,CAAA,CAAC;AACrD,oBAAA,OAAO,IAAI;;;YAEjB,OAAO,KAAK,EAAE;gBACZ,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAA2B,wBAAA,EAAA,QAAQ,CAAgB,aAAA,EAAA,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAE,CAAA,CAAC;;;;QAKpI,IAAI,eAAe,EAAE,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE;YACrF,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAkB,eAAA,EAAA,QAAQ,CAAmC,iCAAA,CAAA,CAAC;AAC9E,YAAA,OAAO,IAAI;;AAGf,QAAA,OAAO,KAAK;;AAEnB;;;;"}