@matperez/coderag 0.1.26 → 0.1.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1 +1 @@
1
- {"version":3,"file":"ast-chunking.d.ts","sourceRoot":"","sources":["../src/ast-chunking.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAUH;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAA;IAC9B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAC1C;AA8CD;;;;;GAKG;AACH,wBAAsB,cAAc,CACnC,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,eAAoB,GAC3B,OAAO,CAAC,SAAS,WAAW,EAAE,CAAC,CAiBjC;AAED;;GAEG;AACH,wBAAsB,oBAAoB,CACzC,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,eAAoB,GAC3B,OAAO,CAAC,SAAS,MAAM,EAAE,CAAC,CAG5B;AAED;;GAEG;AACH,wBAAgB,qBAAqB,IAAI,MAAM,EAAE,CAEhD"}
1
+ {"version":3,"file":"ast-chunking.d.ts","sourceRoot":"","sources":["../src/ast-chunking.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAUH;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAA;IAC9B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAC1C;AA8CD;;;;;GAKG;AACH,wBAAsB,cAAc,CACnC,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,eAAoB,GAC3B,OAAO,CAAC,SAAS,WAAW,EAAE,CAAC,CAwBjC;AAED;;GAEG;AACH,wBAAsB,oBAAoB,CACzC,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,eAAoB,GAC3B,OAAO,CAAC,SAAS,MAAM,EAAE,CAAC,CAG5B;AAED;;GAEG;AACH,wBAAgB,qBAAqB,IAAI,MAAM,EAAE,CAEhD"}
@@ -68,7 +68,9 @@ export async function chunkCodeByAST(code, filePath, options = {}) {
68
68
  }
69
69
  catch (error) {
70
70
  // UnsupportedLanguageError, ChunkingError, or any other: fall back to character chunking
71
- console.error('[WARN] AST chunking failed, falling back to character chunking:', error instanceof Error ? error.message : String(error));
71
+ const err = error instanceof Error ? error : new Error(String(error));
72
+ const causeMsg = err.cause instanceof Error ? ` (cause: ${err.cause.message})` : '';
73
+ console.error('[WARN] AST chunking failed, falling back to character chunking:', filePath, '-', err.message + causeMsg);
72
74
  return createFallbackChunks(code, maxChunkSize);
73
75
  }
74
76
  }
@@ -1 +1 @@
1
- {"version":3,"file":"ast-chunking.js","sourceRoot":"","sources":["../src/ast-chunking.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,YAAY,CAAA;AAE/C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAyB3C,yDAAyD;AACzD,MAAM,8BAA8B,GAAG;IACtC,YAAY;IACZ,YAAY;IACZ,QAAQ;IACR,MAAM;IACN,IAAI;IACJ,MAAM;CACG,CAAA;AAEV;;GAEG;AACH,SAAS,oBAAoB,CAAC,IAAY,EAAE,YAAoB;IAC/D,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,EAAE,EAAE,YAAY,EAAE,CAAC,CAAA;IAChD,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAClC,OAAO;QACP,IAAI,EAAE,MAAM;QACZ,SAAS,EAAE,CAAC;QACZ,OAAO,EAAE,CAAC;QACV,QAAQ,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE;KACtC,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,SAAS,oBAAoB,CAAC,CAAiB;IAC9C,MAAM,WAAW,GAAG,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAA;IACzC,MAAM,IAAI,GAAG,WAAW,EAAE,IAAI,IAAI,OAAO,CAAA;IACzC,OAAO;QACN,OAAO,EAAE,CAAC,CAAC,IAAI;QACf,IAAI;QACJ,SAAS,EAAE,CAAC,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC;QAChC,OAAO,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC;QAC5B,QAAQ,EAAE;YACT,QAAQ,EAAE,KAAK;YACf,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YACnE,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACxC,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,SAAS,EAAE,CAAC,CAAC,SAAS;aACtB,CAAC,CAAC;SACH;KACD,CAAA;AACF,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,IAAY,EACZ,QAAgB,EAChB,UAA2B,EAAE;IAE7B,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAA;IAEjD,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE;YAC9C,YAAY;YACZ,WAAW,EAAE,MAAM;SACnB,CAAC,CAAA;QACF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnD,OAAO,oBAAoB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;QAChD,CAAC;QACD,OAAO,MAAM,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAA;IACxC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,yFAAyF;QACzF,OAAO,CAAC,KAAK,CAAC,iEAAiE,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;QACxI,OAAO,oBAAoB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;IAChD,CAAC;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,IAAY,EACZ,QAAgB,EAChB,UAA2B,EAAE;IAE7B,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAA;IAC5D,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;AAC5C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB;IACpC,OAAO,CAAC,GAAG,8BAA8B,CAAC,CAAA;AAC3C,CAAC"}
1
+ {"version":3,"file":"ast-chunking.js","sourceRoot":"","sources":["../src/ast-chunking.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,YAAY,CAAA;AAE/C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAyB3C,yDAAyD;AACzD,MAAM,8BAA8B,GAAG;IACtC,YAAY;IACZ,YAAY;IACZ,QAAQ;IACR,MAAM;IACN,IAAI;IACJ,MAAM;CACG,CAAA;AAEV;;GAEG;AACH,SAAS,oBAAoB,CAAC,IAAY,EAAE,YAAoB;IAC/D,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,EAAE,EAAE,YAAY,EAAE,CAAC,CAAA;IAChD,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAClC,OAAO;QACP,IAAI,EAAE,MAAM;QACZ,SAAS,EAAE,CAAC;QACZ,OAAO,EAAE,CAAC;QACV,QAAQ,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE;KACtC,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,SAAS,oBAAoB,CAAC,CAAiB;IAC9C,MAAM,WAAW,GAAG,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAA;IACzC,MAAM,IAAI,GAAG,WAAW,EAAE,IAAI,IAAI,OAAO,CAAA;IACzC,OAAO;QACN,OAAO,EAAE,CAAC,CAAC,IAAI;QACf,IAAI;QACJ,SAAS,EAAE,CAAC,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC;QAChC,OAAO,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC;QAC5B,QAAQ,EAAE;YACT,QAAQ,EAAE,KAAK;YACf,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YACnE,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACxC,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,SAAS,EAAE,CAAC,CAAC,SAAS;aACtB,CAAC,CAAC;SACH;KACD,CAAA;AACF,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,IAAY,EACZ,QAAgB,EAChB,UAA2B,EAAE;IAE7B,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAA;IAEjD,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE;YAC9C,YAAY;YACZ,WAAW,EAAE,MAAM;SACnB,CAAC,CAAA;QACF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnD,OAAO,oBAAoB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;QAChD,CAAC;QACD,OAAO,MAAM,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAA;IACxC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,yFAAyF;QACzF,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;QACrE,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,CAAA;QACnF,OAAO,CAAC,KAAK,CACZ,iEAAiE,EACjE,QAAQ,EACR,GAAG,EACH,GAAG,CAAC,OAAO,GAAG,QAAQ,CACtB,CAAA;QACD,OAAO,oBAAoB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;IAChD,CAAC;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,IAAY,EACZ,QAAgB,EAChB,UAA2B,EAAE;IAE7B,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAA;IAC5D,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;AAC5C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB;IACpC,OAAO,CAAC,GAAG,8BAA8B,CAAC,CAAA;AAC3C,CAAC"}
@@ -30,7 +30,9 @@ export declare class CodeTokenizer {
30
30
  initialize(): Promise<void>;
31
31
  private doInitialize;
32
32
  /**
33
- * Tokenize code into terms for TF-IDF indexing
33
+ * Tokenize code into terms for TF-IDF indexing.
34
+ * Uses StarCoder2 when available; falls back to simple word-split when it returns no tokens
35
+ * (e.g. for Go, proto, YAML) so search still works.
34
36
  */
35
37
  tokenize(code: string): Promise<string[]>;
36
38
  /**
@@ -1 +1 @@
1
- {"version":3,"file":"code-tokenizer.d.ts","sourceRoot":"","sources":["../src/code-tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,MAAM,WAAW,SAAS;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,gBAAgB;IAChC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAC1B;AAED;;;;;GAKG;AACH,qBAAa,aAAa;IACzB,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,WAAW,CAAQ;IAC3B,OAAO,CAAC,WAAW,CAA6B;IAChD,OAAO,CAAC,SAAS,CAAQ;gBAEb,OAAO,GAAE,gBAAqB;IAK1C;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;YAcnB,YAAY;IAiB1B;;OAEG;IACG,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IA8B/C;;OAEG;IACG,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAW9D;;OAEG;IACH,OAAO,IAAI,OAAO;CAGlB;AAKD;;GAEG;AACH,wBAAgB,YAAY,IAAI,aAAa,CAK5C;AAED;;;GAGG;AACH,wBAAsB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAG9D;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAG7E;AAED;;GAEG;AACH,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC,CAGzD"}
1
+ {"version":3,"file":"code-tokenizer.d.ts","sourceRoot":"","sources":["../src/code-tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAeH,MAAM,WAAW,SAAS;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,gBAAgB;IAChC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAC1B;AAED;;;;;GAKG;AACH,qBAAa,aAAa;IACzB,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,WAAW,CAAQ;IAC3B,OAAO,CAAC,WAAW,CAA6B;IAChD,OAAO,CAAC,SAAS,CAAQ;gBAEb,OAAO,GAAE,gBAAqB;IAK1C;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;YAcnB,YAAY;IAiB1B;;;;OAIG;IACG,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAiC/C;;OAEG;IACG,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAW9D;;OAEG;IACH,OAAO,IAAI,OAAO;CAGlB;AAKD;;GAEG;AACH,wBAAgB,YAAY,IAAI,aAAa,CAK5C;AAED;;;GAGG;AACH,wBAAsB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAG9D;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAG7E;AAED;;GAEG;AACH,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC,CAGzD"}
@@ -5,6 +5,16 @@
5
5
  * world-class code tokenization quality without requiring the full model.
6
6
  */
7
7
  import { AutoTokenizer } from '@huggingface/transformers';
8
+ /**
9
+ * Simple word/identifier tokenizer fallback when StarCoder2 returns no tokens.
10
+ * Splits on non-alphanumeric and underscore, lowercases, keeps tokens with length > 1.
11
+ */
12
+ function simpleWordTokenize(text) {
13
+ return text
14
+ .split(/[^a-zA-Z0-9_]+/)
15
+ .filter((w) => w.length > 1)
16
+ .map((w) => w.toLowerCase());
17
+ }
8
18
  /**
9
19
  * StarCoder2 Code Tokenizer
10
20
  *
@@ -49,30 +59,40 @@ export class CodeTokenizer {
49
59
  }
50
60
  }
51
61
  /**
52
- * Tokenize code into terms for TF-IDF indexing
62
+ * Tokenize code into terms for TF-IDF indexing.
63
+ * Uses StarCoder2 when available; falls back to simple word-split when it returns no tokens
64
+ * (e.g. for Go, proto, YAML) so search still works.
53
65
  */
54
66
  async tokenize(code) {
55
- if (!this.initialized) {
56
- await this.initialize();
57
- }
58
67
  if (!code || code.trim().length === 0) {
59
68
  return [];
60
69
  }
61
- // Encode with StarCoder2
62
- const encoded = await this.tokenizer(code);
63
- const inputIds = encoded.input_ids.tolist()[0];
64
- // Decode each token ID to get the actual tokens
65
- const tokens = [];
66
- for (const id of inputIds) {
67
- const token = await this.tokenizer.decode([id], {
68
- skip_special_tokens: true,
69
- });
70
- const cleaned = token.trim().toLowerCase();
71
- // Filter: keep tokens with length > 1 (skip single chars and empty)
72
- if (cleaned.length > 1) {
73
- tokens.push(cleaned);
70
+ let tokens = [];
71
+ try {
72
+ if (!this.initialized) {
73
+ await this.initialize();
74
+ }
75
+ // Encode with StarCoder2
76
+ const encoded = await this.tokenizer(code);
77
+ const inputIds = encoded.input_ids.tolist()[0];
78
+ for (const id of inputIds) {
79
+ const token = await this.tokenizer.decode([id], {
80
+ skip_special_tokens: true,
81
+ });
82
+ const cleaned = token.trim().toLowerCase();
83
+ if (cleaned.length > 1) {
84
+ tokens.push(cleaned);
85
+ }
74
86
  }
75
87
  }
88
+ catch (e) {
89
+ // StarCoder2 failed (e.g. OOM, unsupported input)
90
+ console.error('[tokenize] StarCoder2 failed, using fallback:', e.message);
91
+ }
92
+ // Fallback: simple word/identifier split when StarCoder2 returns no tokens (e.g. large chunk, env issue)
93
+ if (tokens.length === 0) {
94
+ tokens = simpleWordTokenize(code);
95
+ }
76
96
  return tokens;
77
97
  }
78
98
  /**
@@ -1 +1 @@
1
- {"version":3,"file":"code-tokenizer.js","sourceRoot":"","sources":["../src/code-tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAA;AAYzD;;;;;GAKG;AACH,MAAM,OAAO,aAAa;IACjB,SAAS,CAAK;IACd,WAAW,GAAG,KAAK,CAAA;IACnB,WAAW,GAAyB,IAAI,CAAA;IACxC,SAAS,CAAQ;IAEzB,YAAY,UAA4B,EAAE;QACzC,8EAA8E;QAC9E,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,wBAAwB,CAAA;IAC/D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACf,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,OAAM;QACP,CAAC;QAED,8CAA8C;QAC9C,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC,WAAW,CAAA;QACxB,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,YAAY,EAAE,CAAA;QACtC,OAAO,IAAI,CAAC,WAAW,CAAA;IACxB,CAAC;IAEO,KAAK,CAAC,YAAY;QACzB,IAAI,CAAC;YACJ,OAAO,CAAC,KAAK,CAAC,mEAAmE,CAAC,CAAA;YAClF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YAE5B,IAAI,CAAC,SAAS,GAAG,MAAM,aAAa,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YAEpE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAA;YACvC,OAAO,CAAC,KAAK,CAAC,iCAAiC,QAAQ,IAAI,CAAC,CAAA;YAE5D,IAAI,CAAC,WAAW,GAAG,IAAI,CAAA;QACxB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAA;YACvB,MAAM,IAAI,KAAK,CAAC,6BAA6B,KAAK,CAAC,OAAO,EAAE,CAAC,CAAA;QAC9D,CAAC;IACF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAY;QAC1B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAA;QACxB,CAAC;QAED,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvC,OAAO,EAAE,CAAA;QACV,CAAC;QAED,yBAAyB;QACzB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;QAC1C,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAA;QAE9C,gDAAgD;QAChD,MAAM,MAAM,GAAa,EAAE,CAAA;QAC3B,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE;gBAC/C,mBAAmB,EAAE,IAAI;aACzB,CAAC,CAAA;YAEF,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;YAC1C,oEAAoE;YACpE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YACrB,CAAC;QACF,CAAC;QAED,OAAO,MAAM,CAAA;IACd,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY,CAAC,IAAY;QAC9B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAA;QAE1C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC5B,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QACpD,CAAC;QAED,OAAO,QAAQ,CAAA;IAChB,CAAC;IAED;;OAEG;IACH,OAAO;QACN,OAAO,IAAI,CAAC,WAAW,CAAA;IACxB,CAAC;CACD;AAED,oCAAoC;AACpC,IAAI,eAAe,GAAyB,IAAI,CAAA;AAEhD;;GAEG;AACH,MAAM,UAAU,YAAY;IAC3B,IAAI,CAAC,eAAe,EAAE,CAAC;QACtB,eAAe,GAAG,IAAI,aAAa,EAAE,CAAA;IACtC,CAAC;IACD,OAAO,eAAe,CAAA;AACvB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,IAAY;IAC1C,MAAM,SAAS,GAAG,YAAY,EAAE,CAAA;IAChC,OAAO,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;AAChC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY;IAC9C,MAAM,SAAS,GAAG,YAAY,EAAE,CAAA;IAChC,OAAO,SAAS,CAAC,YAAY,CAAC,IAAI,CAAC,CAAA;AACpC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB;IACxC,MAAM,SAAS,GAAG,YAAY,EAAE,CAAA;IAChC,MAAM,SAAS,CAAC,UAAU,EAAE,CAAA;AAC7B,CAAC"}
1
+ {"version":3,"file":"code-tokenizer.js","sourceRoot":"","sources":["../src/code-tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAA;AAEzD;;;GAGG;AACH,SAAS,kBAAkB,CAAC,IAAY;IACvC,OAAO,IAAI;SACT,KAAK,CAAC,gBAAgB,CAAC;SACvB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;SAC3B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAA;AAC9B,CAAC;AAYD;;;;;GAKG;AACH,MAAM,OAAO,aAAa;IACjB,SAAS,CAAK;IACd,WAAW,GAAG,KAAK,CAAA;IACnB,WAAW,GAAyB,IAAI,CAAA;IACxC,SAAS,CAAQ;IAEzB,YAAY,UAA4B,EAAE;QACzC,8EAA8E;QAC9E,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,wBAAwB,CAAA;IAC/D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACf,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,OAAM;QACP,CAAC;QAED,8CAA8C;QAC9C,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC,WAAW,CAAA;QACxB,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,YAAY,EAAE,CAAA;QACtC,OAAO,IAAI,CAAC,WAAW,CAAA;IACxB,CAAC;IAEO,KAAK,CAAC,YAAY;QACzB,IAAI,CAAC;YACJ,OAAO,CAAC,KAAK,CAAC,mEAAmE,CAAC,CAAA;YAClF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YAE5B,IAAI,CAAC,SAAS,GAAG,MAAM,aAAa,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YAEpE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAA;YACvC,OAAO,CAAC,KAAK,CAAC,iCAAiC,QAAQ,IAAI,CAAC,CAAA;YAE5D,IAAI,CAAC,WAAW,GAAG,IAAI,CAAA;QACxB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAA;YACvB,MAAM,IAAI,KAAK,CAAC,6BAA6B,KAAK,CAAC,OAAO,EAAE,CAAC,CAAA;QAC9D,CAAC;IACF,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAY;QAC1B,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvC,OAAO,EAAE,CAAA;QACV,CAAC;QAED,IAAI,MAAM,GAAa,EAAE,CAAA;QACzB,IAAI,CAAC;YACJ,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;gBACvB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAA;YACxB,CAAC;YACD,yBAAyB;YACzB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;YAC1C,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAA;YAC9C,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;gBAC3B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE;oBAC/C,mBAAmB,EAAE,IAAI;iBACzB,CAAC,CAAA;gBACF,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;gBAC1C,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACxB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;gBACrB,CAAC;YACF,CAAC;QACF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACZ,kDAAkD;YAClD,OAAO,CAAC,KAAK,CAAC,+CAA+C,EAAG,CAAW,CAAC,OAAO,CAAC,CAAA;QACrF,CAAC;QACD,yGAAyG;QACzG,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;QAClC,CAAC;QACD,OAAO,MAAM,CAAA;IACd,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY,CAAC,IAAY;QAC9B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAA;QAE1C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC5B,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QACpD,CAAC;QAED,OAAO,QAAQ,CAAA;IAChB,CAAC;IAED;;OAEG;IACH,OAAO;QACN,OAAO,IAAI,CAAC,WAAW,CAAA;IACxB,CAAC;CACD;AAED,oCAAoC;AACpC,IAAI,eAAe,GAAyB,IAAI,CAAA;AAEhD;;GAEG;AACH,MAAM,UAAU,YAAY;IAC3B,IAAI,CAAC,eAAe,EAAE,CAAC;QACtB,eAAe,GAAG,IAAI,aAAa,EAAE,CAAA;IACtC,CAAC;IACD,OAAO,eAAe,CAAA;AACvB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,IAAY;IAC1C,MAAM,SAAS,GAAG,YAAY,EAAE,CAAA;IAChC,OAAO,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;AAChC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY;IAC9C,MAAM,SAAS,GAAG,YAAY,EAAE,CAAA;IAChC,OAAO,SAAS,CAAC,YAAY,CAAC,IAAI,CAAC,CAAA;AACpC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB;IACxC,MAAM,SAAS,GAAG,YAAY,EAAE,CAAA;IAChC,MAAM,SAAS,CAAC,UAAU,EAAE,CAAA;AAC7B,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@matperez/coderag",
3
- "version": "0.1.26",
3
+ "version": "0.1.29",
4
4
  "description": "Semantic code search with vector embeddings - RAG-ready for AI assistants",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",