obedding 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +223 -0
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +147 -0
- package/dist/cli.js.map +1 -0
- package/dist/index.d.ts +21 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +136 -0
- package/dist/index.js.map +1 -0
- package/dist/mlx.d.ts +39 -0
- package/dist/mlx.d.ts.map +1 -0
- package/dist/mlx.js +151 -0
- package/dist/mlx.js.map +1 -0
- package/dist/ollama.d.ts +30 -0
- package/dist/ollama.d.ts.map +1 -0
- package/dist/ollama.js +101 -0
- package/dist/ollama.js.map +1 -0
- package/dist/scanner.d.ts +32 -0
- package/dist/scanner.d.ts.map +1 -0
- package/dist/scanner.js +104 -0
- package/dist/scanner.js.map +1 -0
- package/dist/search.d.ts +36 -0
- package/dist/search.d.ts.map +1 -0
- package/dist/search.js +115 -0
- package/dist/search.js.map +1 -0
- package/dist/storage.d.ts +82 -0
- package/dist/storage.d.ts.map +1 -0
- package/dist/storage.js +183 -0
- package/dist/storage.js.map +1 -0
- package/dist/utils.d.ts +25 -0
- package/dist/utils.d.ts.map +1 -0
- package/dist/utils.js +69 -0
- package/dist/utils.js.map +1 -0
- package/package.json +57 -0
package/dist/mlx.js
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MLX Embedding API client for generating embeddings
|
|
3
|
+
* Compatible with OpenAI-style /v1/embeddings endpoint
|
|
4
|
+
*/
|
|
5
|
+
const MLX_BASE_URL = process.env.MLX_BASE_URL || 'http://localhost:28100';
|
|
6
|
+
const DEFAULT_MODEL = 'mlx-community/Qwen3-Embedding-0.6B-4bit-DWQ';
|
|
7
|
+
/**
|
|
8
|
+
* Check if MLX embedding server is running
|
|
9
|
+
*/
|
|
10
|
+
export async function checkMLXConnection() {
|
|
11
|
+
try {
|
|
12
|
+
// Try a minimal embedding request as health check
|
|
13
|
+
const response = await fetch(`${MLX_BASE_URL}/v1/embeddings`, {
|
|
14
|
+
method: 'POST',
|
|
15
|
+
headers: { 'Content-Type': 'application/json' },
|
|
16
|
+
body: JSON.stringify({
|
|
17
|
+
input: 'test',
|
|
18
|
+
model: DEFAULT_MODEL
|
|
19
|
+
}),
|
|
20
|
+
signal: AbortSignal.timeout(5000)
|
|
21
|
+
});
|
|
22
|
+
return response.ok;
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
return false;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Generate embedding for text using MLX server
|
|
30
|
+
*/
|
|
31
|
+
export async function generateEmbedding(text, model = DEFAULT_MODEL) {
|
|
32
|
+
const response = await fetch(`${MLX_BASE_URL}/v1/embeddings`, {
|
|
33
|
+
method: 'POST',
|
|
34
|
+
headers: {
|
|
35
|
+
'Content-Type': 'application/json',
|
|
36
|
+
},
|
|
37
|
+
body: JSON.stringify({
|
|
38
|
+
input: [text], // Use array format for consistent fixed-length embeddings
|
|
39
|
+
model
|
|
40
|
+
})
|
|
41
|
+
});
|
|
42
|
+
if (!response.ok) {
|
|
43
|
+
let errorMessage = response.statusText;
|
|
44
|
+
try {
|
|
45
|
+
const error = await response.json();
|
|
46
|
+
if (error.error) {
|
|
47
|
+
errorMessage = error.error;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
catch {
|
|
51
|
+
try {
|
|
52
|
+
errorMessage = await response.text();
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
// Use status text
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
throw new Error(`MLX API error: ${errorMessage}`);
|
|
59
|
+
}
|
|
60
|
+
const data = await response.json();
|
|
61
|
+
if (!data.data || !Array.isArray(data.data) || data.data.length === 0) {
|
|
62
|
+
throw new Error('Invalid embedding response from MLX server');
|
|
63
|
+
}
|
|
64
|
+
const embedding = data.data[0].embedding;
|
|
65
|
+
if (!embedding || !Array.isArray(embedding)) {
|
|
66
|
+
throw new Error('Invalid embedding format in MLX response');
|
|
67
|
+
}
|
|
68
|
+
// Truncate or pad to ensure fixed-length embeddings (2048 dimensions)
|
|
69
|
+
const fixedLength = 2048;
|
|
70
|
+
if (embedding.length > fixedLength) {
|
|
71
|
+
return embedding.slice(0, fixedLength);
|
|
72
|
+
}
|
|
73
|
+
else if (embedding.length < fixedLength) {
|
|
74
|
+
// Pad with zeros (shouldn't happen, but handle it)
|
|
75
|
+
const padded = new Array(fixedLength).fill(0);
|
|
76
|
+
for (let i = 0; i < embedding.length; i++) {
|
|
77
|
+
padded[i] = embedding[i];
|
|
78
|
+
}
|
|
79
|
+
return padded;
|
|
80
|
+
}
|
|
81
|
+
return embedding;
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Batch generate embeddings for multiple texts
|
|
85
|
+
*/
|
|
86
|
+
export async function generateEmbeddings(texts, model = DEFAULT_MODEL, onProgress) {
|
|
87
|
+
const embeddings = [];
|
|
88
|
+
// Process in batches for efficiency
|
|
89
|
+
const batchSize = 4;
|
|
90
|
+
for (let i = 0; i < texts.length; i += batchSize) {
|
|
91
|
+
const batch = texts.slice(i, i + batchSize);
|
|
92
|
+
const response = await fetch(`${MLX_BASE_URL}/v1/embeddings`, {
|
|
93
|
+
method: 'POST',
|
|
94
|
+
headers: {
|
|
95
|
+
'Content-Type': 'application/json',
|
|
96
|
+
},
|
|
97
|
+
body: JSON.stringify({
|
|
98
|
+
input: batch,
|
|
99
|
+
model
|
|
100
|
+
})
|
|
101
|
+
});
|
|
102
|
+
if (!response.ok) {
|
|
103
|
+
throw new Error(`MLX API error: ${response.statusText}`);
|
|
104
|
+
}
|
|
105
|
+
const data = await response.json();
|
|
106
|
+
if (!data.data || !Array.isArray(data.data)) {
|
|
107
|
+
throw new Error('Invalid embedding response from MLX server');
|
|
108
|
+
}
|
|
109
|
+
for (const item of data.data) {
|
|
110
|
+
// Truncate or pad to ensure fixed-length embeddings (2048 dimensions)
|
|
111
|
+
const embedding = item.embedding;
|
|
112
|
+
const fixedLength = 2048;
|
|
113
|
+
if (embedding.length > fixedLength) {
|
|
114
|
+
// Truncate to first 2048 dimensions
|
|
115
|
+
embeddings.push(embedding.slice(0, fixedLength));
|
|
116
|
+
}
|
|
117
|
+
else if (embedding.length < fixedLength) {
|
|
118
|
+
// Pad with zeros (shouldn't happen, but handle it)
|
|
119
|
+
const padded = new Array(fixedLength).fill(0);
|
|
120
|
+
for (let i = 0; i < embedding.length; i++) {
|
|
121
|
+
padded[i] = embedding[i];
|
|
122
|
+
}
|
|
123
|
+
embeddings.push(padded);
|
|
124
|
+
}
|
|
125
|
+
else {
|
|
126
|
+
embeddings.push(embedding);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
if (onProgress) {
|
|
130
|
+
onProgress(Math.min(i + batchSize, texts.length), texts.length);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
return embeddings;
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Get model information (basic, MLX doesn't expose model info like Ollama)
|
|
137
|
+
*/
|
|
138
|
+
export async function getModelInfo(model = DEFAULT_MODEL) {
|
|
139
|
+
try {
|
|
140
|
+
// Try a test embedding to get dimensions
|
|
141
|
+
const testEmbedding = await generateEmbedding('test', model);
|
|
142
|
+
return {
|
|
143
|
+
name: model,
|
|
144
|
+
dimensions: testEmbedding.length
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
catch {
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
//# sourceMappingURL=mlx.js.map
|
package/dist/mlx.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mlx.js","sourceRoot":"","sources":["../src/mlx.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,wBAAwB,CAAC;AAC1E,MAAM,aAAa,GAAG,6CAA6C,CAAC;AAmBpE;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACtC,IAAI,CAAC;QACH,kDAAkD;QAClD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,YAAY,gBAAgB,EAAE;YAC5D,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK,EAAE,MAAM;gBACb,KAAK,EAAE,aAAa;aACrB,CAAC;YACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC;SAClC,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC,EAAE,CAAC;IACrB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,QAAgB,aAAa;IAE7B,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,YAAY,gBAAgB,EAAE;QAC5D,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACP,cAAc,EAAE,kBAAkB;SACnC;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACnB,KAAK,EAAE,CAAC,IAAI,CAAC,EAAG,0DAA0D;YAC1E,KAAK;SACN,CAAC;KACH,CAAC,CAAC;IAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,IAAI,YAAY,GAAG,QAAQ,CAAC,UAAU,CAAC;QACvC,IAAI,CAAC;YACH,MAAM,KAAK,GAAa,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAC9C,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;gBAChB,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC;YAC7B,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC;gBACH,YAAY,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACvC,CAAC;YAAC,MAAM,CAAC;gBACP,kBAAkB;YACpB,CAAC;QACH,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,kBAAkB,YAAY,EAAE,CAAC,CAAC;IACpD,CAAC;IAED,MAAM,IAAI,GAAsB,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IAEtD,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACzC,IAAI,CAAC,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAC9D,CAAC;IAED,sEAAsE;IACtE,MAAM,WAAW,GAAG,IAAI,CAAC;IACzB,IAAI,SAAS,CAAC,MAAM,GAAG,WAAW,EAAE,CAAC;QACnC,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;IACzC,CAAC;SAAM,IAAI,SAAS,CAAC,MAAM,GAAG,WAAW,EAAE,CAAC;QAC1C,mDAAmD;QACnD,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QAC3B,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,KAAe,EACf,QAAgB,aAAa,EAC7B,UAAqD;IAErD,MAAM,UAAU,GAAe,EAAE,CAAC;IAElC,oCAAoC;IACpC,MAAM,SAAS,GAAG,CAAC,CAAC;IACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QACjD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;QAC5C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,YAAY,gBAAgB,EAAE;YAC5D,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;aACnC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK,EAAE,KAAK;gBACZ,KAAK;aACN,CAAC;SACH,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,kBAAkB,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,IAAI,GAAsB,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAEtD,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5C,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAChE,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAC7B,sEAAsE;YACtE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;YACjC,MAAM,WAAW,GAAG,IAAI,CAAC;YAEzB,IAAI,SAAS,CAAC,MAAM,GAAG,WAAW,EAAE,CAAC;gBACnC,oCAAoC;gBACpC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC;YACnD,CAAC;iBAAM,IAAI,SAAS,CAAC,MAAM,GAAG,WAAW,EAAE,CAAC;gBAC1C,mDAAmD;gBACnD,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC1C,MAAM,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;gBAC3B,CAAC;gBACD,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC1B,CAAC;iBAAM,CAAC;gBACN,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC7B,CAAC;QACH,CAAC;QAED,IAAI,UAAU,EAAE,CAAC;YACf,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAgB,aAAa;IAI9D,IAAI,CAAC;QACH,yCAAyC;QACzC,MAAM,aAAa,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAC7D,OAAO;YACL,IAAI,EAAE,KAAK;YACX,UAAU,EAAE,aAAa,CAAC,MAAM;SACjC,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
|
package/dist/ollama.d.ts
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ollama API client for generating embeddings
|
|
3
|
+
*/
|
|
4
|
+
export interface EmbeddingResponse {
|
|
5
|
+
embedding: number[];
|
|
6
|
+
}
|
|
7
|
+
export interface OllamaError {
|
|
8
|
+
error: string;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Check if Ollama is running
|
|
12
|
+
*/
|
|
13
|
+
export declare function checkOllamaConnection(): Promise<boolean>;
|
|
14
|
+
/**
|
|
15
|
+
* Generate embedding for text using Ollama
|
|
16
|
+
*/
|
|
17
|
+
export declare function generateEmbedding(text: string, model?: string): Promise<number[]>;
|
|
18
|
+
/**
|
|
19
|
+
* Batch generate embeddings for multiple texts
|
|
20
|
+
*/
|
|
21
|
+
export declare function generateEmbeddings(texts: string[], model?: string, onProgress?: (current: number, total: number) => void): Promise<number[][]>;
|
|
22
|
+
/**
|
|
23
|
+
* Get model information
|
|
24
|
+
*/
|
|
25
|
+
export declare function getModelInfo(model?: string): Promise<{
|
|
26
|
+
name: string;
|
|
27
|
+
size: number;
|
|
28
|
+
digest: string;
|
|
29
|
+
} | null>;
|
|
30
|
+
//# sourceMappingURL=ollama.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ollama.d.ts","sourceRoot":"","sources":["../src/ollama.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,wBAAsB,qBAAqB,IAAI,OAAO,CAAC,OAAO,CAAC,CAU9D;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,MAAM,EACZ,KAAK,GAAE,MAAsB,GAC5B,OAAO,CAAC,MAAM,EAAE,CAAC,CAuCnB;AAED;;GAEG;AACH,wBAAsB,kBAAkB,CACtC,KAAK,EAAE,MAAM,EAAE,EACf,KAAK,GAAE,MAAsB,EAC7B,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GACpD,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAarB;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,KAAK,GAAE,MAAsB,GAAG,OAAO,CAAC;IACzE,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;CAChB,GAAG,IAAI,CAAC,CAuBR"}
|
package/dist/ollama.js
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ollama API client for generating embeddings
|
|
3
|
+
*/
|
|
4
|
+
const OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL || 'http://localhost:11434';
|
|
5
|
+
const DEFAULT_MODEL = 'qwen3-embedding:0.6b';
|
|
6
|
+
/**
|
|
7
|
+
* Check if Ollama is running
|
|
8
|
+
*/
|
|
9
|
+
export async function checkOllamaConnection() {
|
|
10
|
+
try {
|
|
11
|
+
const response = await fetch(`${OLLAMA_BASE_URL}/api/tags`, {
|
|
12
|
+
method: 'GET',
|
|
13
|
+
signal: AbortSignal.timeout(5000)
|
|
14
|
+
});
|
|
15
|
+
return response.ok;
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
return false;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Generate embedding for text using Ollama
|
|
23
|
+
*/
|
|
24
|
+
export async function generateEmbedding(text, model = DEFAULT_MODEL) {
|
|
25
|
+
const response = await fetch(`${OLLAMA_BASE_URL}/api/embeddings`, {
|
|
26
|
+
method: 'POST',
|
|
27
|
+
headers: {
|
|
28
|
+
'Content-Type': 'application/json',
|
|
29
|
+
},
|
|
30
|
+
body: JSON.stringify({
|
|
31
|
+
model,
|
|
32
|
+
prompt: text
|
|
33
|
+
})
|
|
34
|
+
});
|
|
35
|
+
if (!response.ok) {
|
|
36
|
+
// Try to parse as JSON first, fall back to text
|
|
37
|
+
let errorMessage = response.statusText;
|
|
38
|
+
try {
|
|
39
|
+
const error = await response.json();
|
|
40
|
+
if (error.error) {
|
|
41
|
+
errorMessage = error.error;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
catch {
|
|
45
|
+
// Not JSON, use status text or try reading body as text
|
|
46
|
+
try {
|
|
47
|
+
errorMessage = await response.text();
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
// Use status text
|
|
51
|
+
errorMessage = response.statusText;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
throw new Error(`Ollama API error: ${errorMessage}`);
|
|
55
|
+
}
|
|
56
|
+
const data = await response.json();
|
|
57
|
+
if (!data.embedding || !Array.isArray(data.embedding)) {
|
|
58
|
+
throw new Error('Invalid embedding response from Ollama');
|
|
59
|
+
}
|
|
60
|
+
return data.embedding;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Batch generate embeddings for multiple texts
|
|
64
|
+
*/
|
|
65
|
+
export async function generateEmbeddings(texts, model = DEFAULT_MODEL, onProgress) {
|
|
66
|
+
const embeddings = [];
|
|
67
|
+
for (let i = 0; i < texts.length; i++) {
|
|
68
|
+
const embedding = await generateEmbedding(texts[i], model);
|
|
69
|
+
embeddings.push(embedding);
|
|
70
|
+
if (onProgress) {
|
|
71
|
+
onProgress(i + 1, texts.length);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
return embeddings;
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Get model information
|
|
78
|
+
*/
|
|
79
|
+
export async function getModelInfo(model = DEFAULT_MODEL) {
|
|
80
|
+
try {
|
|
81
|
+
const response = await fetch(`${OLLAMA_BASE_URL}/api/tags`);
|
|
82
|
+
if (!response.ok) {
|
|
83
|
+
return null;
|
|
84
|
+
}
|
|
85
|
+
const data = await response.json();
|
|
86
|
+
const models = data.models || [];
|
|
87
|
+
const modelInfo = models.find((m) => m.name === model || m.name.startsWith(model));
|
|
88
|
+
if (modelInfo) {
|
|
89
|
+
return {
|
|
90
|
+
name: modelInfo.name,
|
|
91
|
+
size: modelInfo.size || 0,
|
|
92
|
+
digest: modelInfo.digest || ''
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
return null;
|
|
96
|
+
}
|
|
97
|
+
catch {
|
|
98
|
+
return null;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
//# sourceMappingURL=ollama.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ollama.js","sourceRoot":"","sources":["../src/ollama.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,wBAAwB,CAAC;AAChF,MAAM,aAAa,GAAG,sBAAsB,CAAC;AAU7C;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB;IACzC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,eAAe,WAAW,EAAE;YAC1D,MAAM,EAAE,KAAK;YACb,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC;SAClC,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC,EAAE,CAAC;IACrB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,QAAgB,aAAa;IAE7B,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,eAAe,iBAAiB,EAAE;QAChE,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACP,cAAc,EAAE,kBAAkB;SACnC;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACnB,KAAK;YACL,MAAM,EAAE,IAAI;SACb,CAAC;KACH,CAAC,CAAC;IAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,gDAAgD;QAChD,IAAI,YAAY,GAAG,QAAQ,CAAC,UAAU,CAAC;QACvC,IAAI,CAAC;YACH,MAAM,KAAK,GAAgB,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACjD,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;gBAChB,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC;YAC7B,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,wDAAwD;YACxD,IAAI,CAAC;gBACH,YAAY,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACvC,CAAC;YAAC,MAAM,CAAC;gBACP,kBAAkB;gBAClB,YAAY,GAAG,QAAQ,CAAC,UAAU,CAAC;YACrC,CAAC;QACH,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,qBAAqB,YAAY,EAAE,CAAC,CAAC;IACvD,CAAC;IAED,MAAM,IAAI,GAAsB,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IAEtD,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACtD,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC5D,CAAC;IAED,OAAO,IAAI,CAAC,SAAS,CAAC;AACxB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,KAAe,EACf,QAAgB,aAAa,EAC7B,UAAqD;IAErD,MAAM,UAAU,GAAe,EAAE,CAAC;IAElC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QAC3D,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAE3B,IAAI,UAAU,EAAE,CAAC;YACf,UAAU,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAgB,aAAa;IAK9D,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,eAAe,WAAW,CAAC,CAAC;QAC5D,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;QAEjC,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;QACxF,IAAI,SAAS,EAAE,CAAC;YACd,OAAO;gBACL,IAAI,EAAE,SAAS,CAAC,IAAI;gBACpB,IAAI,EAAE,SAAS,CAAC,IAAI,IAAI,CAAC;gBACzB,MAAM,EAAE,SAAS,CAAC,MAAM,IAAI,EAAE;aAC/B,CAAC;QACJ,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
export interface NoteFile {
|
|
2
|
+
relativePath: string;
|
|
3
|
+
fullPath: string;
|
|
4
|
+
content: string;
|
|
5
|
+
frontmatter: {
|
|
6
|
+
data: any;
|
|
7
|
+
content: string;
|
|
8
|
+
};
|
|
9
|
+
modified: Date;
|
|
10
|
+
size: number;
|
|
11
|
+
}
|
|
12
|
+
export interface NoteMetadata {
|
|
13
|
+
type?: string;
|
|
14
|
+
repo?: string;
|
|
15
|
+
context?: string;
|
|
16
|
+
modified?: string;
|
|
17
|
+
tags?: string[];
|
|
18
|
+
title?: string;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Scan Obsidian Projects folder for markdown files
|
|
22
|
+
*/
|
|
23
|
+
export declare function scanObsidianVault(vaultPath: string, patterns?: string[]): Promise<NoteFile[]>;
|
|
24
|
+
/**
|
|
25
|
+
* Extract metadata from frontmatter
|
|
26
|
+
*/
|
|
27
|
+
export declare function extractMetadata(note: NoteFile, vaultPath: string): NoteMetadata;
|
|
28
|
+
/**
|
|
29
|
+
* Get note hash for incremental updates
|
|
30
|
+
*/
|
|
31
|
+
export declare function getNoteHash(note: NoteFile): string;
|
|
32
|
+
//# sourceMappingURL=scanner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scanner.d.ts","sourceRoot":"","sources":["../src/scanner.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,QAAQ;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE;QACX,IAAI,EAAE,GAAG,CAAC;QACV,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;IACF,QAAQ,EAAE,IAAI,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CACrC,SAAS,EAAE,MAAM,EACjB,QAAQ,GAAE,MAAM,EAAgB,GAC/B,OAAO,CAAC,QAAQ,EAAE,CAAC,CAmDrB;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,GAAG,YAAY,CAiC/E;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,QAAQ,GAAG,MAAM,CAGlD"}
|
package/dist/scanner.js
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import * as fs from 'fs/promises';
|
|
2
|
+
import * as path from 'path';
|
|
3
|
+
import { glob } from 'glob';
|
|
4
|
+
import matter from 'gray-matter';
|
|
5
|
+
import { hashContent } from './utils.js';
|
|
6
|
+
/**
|
|
7
|
+
* Scan Obsidian Projects folder for markdown files
|
|
8
|
+
*/
|
|
9
|
+
export async function scanObsidianVault(vaultPath, patterns = ['**/*.md']) {
|
|
10
|
+
// Normalize path
|
|
11
|
+
const normalizedVaultPath = path.resolve(vaultPath);
|
|
12
|
+
// Check if vault exists
|
|
13
|
+
try {
|
|
14
|
+
await fs.access(normalizedVaultPath);
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
throw new Error(`Vault path does not exist: ${normalizedVaultPath}`);
|
|
18
|
+
}
|
|
19
|
+
// Find all markdown files
|
|
20
|
+
const files = [];
|
|
21
|
+
for (const pattern of patterns) {
|
|
22
|
+
const matches = await glob(pattern, {
|
|
23
|
+
cwd: normalizedVaultPath,
|
|
24
|
+
absolute: false,
|
|
25
|
+
ignore: ['**/.obsidian/**', '**/node_modules/**']
|
|
26
|
+
});
|
|
27
|
+
files.push(...matches);
|
|
28
|
+
}
|
|
29
|
+
// Remove duplicates and sort
|
|
30
|
+
const uniqueFiles = [...new Set(files)].sort();
|
|
31
|
+
// Read all files
|
|
32
|
+
const notes = [];
|
|
33
|
+
for (const relativePath of uniqueFiles) {
|
|
34
|
+
try {
|
|
35
|
+
const fullPath = path.join(normalizedVaultPath, relativePath);
|
|
36
|
+
const content = await fs.readFile(fullPath, 'utf-8');
|
|
37
|
+
const stats = await fs.stat(fullPath);
|
|
38
|
+
// Parse frontmatter
|
|
39
|
+
const frontmatter = matter(content);
|
|
40
|
+
notes.push({
|
|
41
|
+
relativePath,
|
|
42
|
+
fullPath,
|
|
43
|
+
content,
|
|
44
|
+
frontmatter,
|
|
45
|
+
modified: stats.mtime,
|
|
46
|
+
size: stats.size
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
catch (error) {
|
|
50
|
+
// Skip files that can't be read
|
|
51
|
+
console.warn(`Warning: Could not read file ${relativePath}: ${error.message}`);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
return notes;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Extract metadata from frontmatter
|
|
58
|
+
*/
|
|
59
|
+
export function extractMetadata(note, vaultPath) {
|
|
60
|
+
const { data } = note.frontmatter;
|
|
61
|
+
const metadata = {};
|
|
62
|
+
// Extract from frontmatter
|
|
63
|
+
if (data.type)
|
|
64
|
+
metadata.type = data.type;
|
|
65
|
+
if (data.repo)
|
|
66
|
+
metadata.repo = data.repo;
|
|
67
|
+
if (data.context)
|
|
68
|
+
metadata.context = data.context;
|
|
69
|
+
if (data.date)
|
|
70
|
+
metadata.modified = data.date;
|
|
71
|
+
if (data.tags)
|
|
72
|
+
metadata.tags = Array.isArray(data.tags) ? data.tags : [data.tags];
|
|
73
|
+
// Try to infer from path
|
|
74
|
+
const pathParts = note.relativePath.split(path.sep);
|
|
75
|
+
// Path structure: Projects/{type}/{repo}/{context}/{filename}.md
|
|
76
|
+
if (pathParts[0] === 'Projects' && pathParts.length >= 3) {
|
|
77
|
+
if (!metadata.type && pathParts[1])
|
|
78
|
+
metadata.type = pathParts[1];
|
|
79
|
+
if (!metadata.repo && pathParts[2])
|
|
80
|
+
metadata.repo = pathParts[2];
|
|
81
|
+
if (!metadata.context && pathParts[3])
|
|
82
|
+
metadata.context = pathParts[3];
|
|
83
|
+
}
|
|
84
|
+
// Extract title from content (first heading)
|
|
85
|
+
if (!metadata.title) {
|
|
86
|
+
const titleMatch = note.frontmatter.content.match(/^#\s+(.+)$/m);
|
|
87
|
+
if (titleMatch) {
|
|
88
|
+
metadata.title = titleMatch[1].trim();
|
|
89
|
+
}
|
|
90
|
+
else {
|
|
91
|
+
// Use filename as title
|
|
92
|
+
metadata.title = path.basename(note.relativePath, '.md');
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
return metadata;
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Get note hash for incremental updates
|
|
99
|
+
*/
|
|
100
|
+
export function getNoteHash(note) {
|
|
101
|
+
const content = note.frontmatter.content; // Content without frontmatter
|
|
102
|
+
return hashContent(content);
|
|
103
|
+
}
|
|
104
|
+
//# sourceMappingURL=scanner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scanner.js","sourceRoot":"","sources":["../src/scanner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAuBzC;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,SAAiB,EACjB,WAAqB,CAAC,SAAS,CAAC;IAEhC,iBAAiB;IACjB,MAAM,mBAAmB,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAEpD,wBAAwB;IACxB,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,8BAA8B,mBAAmB,EAAE,CAAC,CAAC;IACvE,CAAC;IAED,0BAA0B;IAC1B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE;YAClC,GAAG,EAAE,mBAAmB;YACxB,QAAQ,EAAE,KAAK;YACf,MAAM,EAAE,CAAC,iBAAiB,EAAE,oBAAoB,CAAC;SAClD,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;IACzB,CAAC;IAED,6BAA6B;IAC7B,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAE/C,iBAAiB;IACjB,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,KAAK,MAAM,YAAY,IAAI,WAAW,EAAE,CAAC;QACvC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,YAAY,CAAC,CAAC;YAC9D,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACrD,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAEtC,oBAAoB;YACpB,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;YAEpC,KAAK,CAAC,IAAI,CAAC;gBACT,YAAY;gBACZ,QAAQ;gBACR,OAAO;gBACP,WAAW;gBACX,QAAQ,EAAE,KAAK,CAAC,KAAK;gBACrB,IAAI,EAAE,KAAK,CAAC,IAAI;aACjB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,gCAAgC;YAChC,OAAO,CAAC,IAAI,CAAC,gCAAgC,YAAY,KAAM,KAAe,CAAC,OAAO,EAAE,CAAC,CAAC;QAC5F,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,IAAc,EAAE,SAAiB;IAC/D,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC;IAClC,MAAM,QAAQ,GAAiB,EAAE,CAAC;IAElC,2BAA2B;IAC3B,IAAI,IAAI,CAAC,IAAI;QAAE,QAAQ,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;IACzC,IAAI,IAAI,CAAC,IAAI;QAAE,QAAQ,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;IACzC,IAAI,IAAI,CAAC,OAAO;QAAE,QAAQ,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;IAClD,IAAI,IAAI,CAAC,IAAI;QAAE,QAAQ,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC;IAC7C,IAAI,IAAI,CAAC,IAAI;QAAE,QAAQ,CAAC,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAElF,yBAAyB;IACzB,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEpD,iEAAiE;IACjE,IAAI,SAAS,CAAC,CAAC,CAAC,KAAK,UAAU,IAAI,SAAS,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACzD,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,SAAS,CAAC,CAAC,CAAC;YAAE,QAAQ,CAAC,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QACjE,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,SAAS,CAAC,CAAC,CAAC;YAAE,QAAQ,CAAC,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QACjE,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,SAAS,CAAC,CAAC,CAAC;YAAE,QAAQ,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC;IAED,6CAA6C;IAC7C,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACpB,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QACjE,IAAI,UAAU,EAAE,CAAC;YACf,QAAQ,CAAC,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACxC,CAAC;aAAM,CAAC;YACN,wBAAwB;YACxB,QAAQ,CAAC,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,IAAc;IACxC,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,8BAA8B;IACxE,OAAO,WAAW,CAAC,OAAO,CAAC,CAAC;AAC9B,CAAC"}
|
package/dist/search.d.ts
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
export interface SearchResult {
|
|
2
|
+
path: string;
|
|
3
|
+
vault_path: string;
|
|
4
|
+
score: number;
|
|
5
|
+
metadata: {
|
|
6
|
+
type?: string;
|
|
7
|
+
repo?: string;
|
|
8
|
+
context?: string;
|
|
9
|
+
modified?: string;
|
|
10
|
+
tags?: string[];
|
|
11
|
+
title?: string;
|
|
12
|
+
};
|
|
13
|
+
excerpt: string;
|
|
14
|
+
}
|
|
15
|
+
export interface SearchOptions {
|
|
16
|
+
query: string;
|
|
17
|
+
storagePath: string;
|
|
18
|
+
topK?: number;
|
|
19
|
+
model?: string;
|
|
20
|
+
minScore?: number;
|
|
21
|
+
}
|
|
22
|
+
export interface SearchResponse {
|
|
23
|
+
query: string;
|
|
24
|
+
results: SearchResult[];
|
|
25
|
+
totalFound: number;
|
|
26
|
+
duration: number;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Search notes using semantic similarity
|
|
30
|
+
*/
|
|
31
|
+
export declare function searchNotes(options: SearchOptions): Promise<SearchResponse>;
|
|
32
|
+
/**
|
|
33
|
+
* Format search results for display
|
|
34
|
+
*/
|
|
35
|
+
export declare function formatSearchResults(response: SearchResponse): string;
|
|
36
|
+
//# sourceMappingURL=search.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE;QACR,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;QAChB,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,wBAAsB,WAAW,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,cAAc,CAAC,CA0EjF;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,cAAc,GAAG,MAAM,CAoCpE"}
|
package/dist/search.js
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { generateEmbedding, checkMLXConnection } from './mlx.js';
|
|
2
|
+
import { StorageManager } from './storage.js';
|
|
3
|
+
import { cosineSimilarity, truncateToTokens } from './utils.js';
|
|
4
|
+
/**
|
|
5
|
+
* Search notes using semantic similarity
|
|
6
|
+
*/
|
|
7
|
+
export async function searchNotes(options) {
|
|
8
|
+
const startTime = Date.now();
|
|
9
|
+
const { query, storagePath, topK = 10, model = 'mlx-community/Qwen3-Embedding-0.6B-4bit-DWQ', minScore = 0.0 } = options;
|
|
10
|
+
// Check MLX connection
|
|
11
|
+
const isConnected = await checkMLXConnection();
|
|
12
|
+
if (!isConnected) {
|
|
13
|
+
throw new Error('Cannot connect to MLX embedding server. Make sure it is running at http://localhost:28100');
|
|
14
|
+
}
|
|
15
|
+
// Load storage
|
|
16
|
+
const storage = new StorageManager(storagePath);
|
|
17
|
+
await storage.initialize();
|
|
18
|
+
const notes = storage.getNotes();
|
|
19
|
+
if (notes.length === 0) {
|
|
20
|
+
return {
|
|
21
|
+
query,
|
|
22
|
+
results: [],
|
|
23
|
+
totalFound: 0,
|
|
24
|
+
duration: (Date.now() - startTime) / 1000
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
// Generate query embedding
|
|
28
|
+
const truncatedQuery = truncateToTokens(query, 8000);
|
|
29
|
+
const queryEmbedding = await generateEmbedding(truncatedQuery, model);
|
|
30
|
+
// Calculate similarities
|
|
31
|
+
const results = [];
|
|
32
|
+
for (const note of notes) {
|
|
33
|
+
// Skip notes with invalid embeddings
|
|
34
|
+
if (!note.embedding || note.embedding.length === 0) {
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
// Skip notes with incompatible embedding dimensions
|
|
38
|
+
if (note.embedding.length !== queryEmbedding.length) {
|
|
39
|
+
console.warn(`Warning: Skipping ${note.path} with incompatible embedding dimension (${note.embedding.length} vs ${queryEmbedding.length})`);
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
const score = cosineSimilarity(queryEmbedding, note.embedding);
|
|
43
|
+
if (score >= minScore) {
|
|
44
|
+
results.push({ note, score });
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
// Sort by score (descending)
|
|
48
|
+
results.sort((a, b) => b.score - a.score);
|
|
49
|
+
// Get top-K
|
|
50
|
+
const topResults = results.slice(0, topK);
|
|
51
|
+
// Format results
|
|
52
|
+
const formattedResults = topResults.map(({ note, score }) => ({
|
|
53
|
+
path: note.path,
|
|
54
|
+
vault_path: note.vault_path,
|
|
55
|
+
score: Number(score.toFixed(4)),
|
|
56
|
+
metadata: note.metadata,
|
|
57
|
+
excerpt: note.excerpt
|
|
58
|
+
}));
|
|
59
|
+
const duration = (Date.now() - startTime) / 1000;
|
|
60
|
+
return {
|
|
61
|
+
query,
|
|
62
|
+
results: formattedResults,
|
|
63
|
+
totalFound: results.length,
|
|
64
|
+
duration
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Format search results for display
|
|
69
|
+
*/
|
|
70
|
+
export function formatSearchResults(response) {
|
|
71
|
+
const lines = [];
|
|
72
|
+
lines.push(`\nQuery: "${response.query}"`);
|
|
73
|
+
lines.push(`Found ${response.totalFound} results in ${response.duration.toFixed(3)}s`);
|
|
74
|
+
lines.push(`Showing top ${response.results.length} results:\n`);
|
|
75
|
+
if (response.results.length === 0) {
|
|
76
|
+
lines.push(' No results found.');
|
|
77
|
+
}
|
|
78
|
+
for (let i = 0; i < response.results.length; i++) {
|
|
79
|
+
const result = response.results[i];
|
|
80
|
+
lines.push(` [${i + 1}] ${formatScore(result.score)} ${result.path}`);
|
|
81
|
+
if (result.metadata.title) {
|
|
82
|
+
lines.push(` Title: ${result.metadata.title}`);
|
|
83
|
+
}
|
|
84
|
+
if (result.metadata.repo || result.metadata.context) {
|
|
85
|
+
const context = [result.metadata.repo, result.metadata.context].filter(Boolean).join(' / ');
|
|
86
|
+
lines.push(` Context: ${context}`);
|
|
87
|
+
}
|
|
88
|
+
if (result.metadata.tags && result.metadata.tags.length > 0) {
|
|
89
|
+
lines.push(` Tags: ${result.metadata.tags.join(', ')}`);
|
|
90
|
+
}
|
|
91
|
+
if (result.excerpt) {
|
|
92
|
+
lines.push(` ${result.excerpt}`);
|
|
93
|
+
}
|
|
94
|
+
lines.push('');
|
|
95
|
+
}
|
|
96
|
+
return lines.join('\n');
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Format similarity score with color indicators
|
|
100
|
+
*/
|
|
101
|
+
function formatScore(score) {
|
|
102
|
+
if (score >= 0.8) {
|
|
103
|
+
return `●●● ${(score * 100).toFixed(1)}%`;
|
|
104
|
+
}
|
|
105
|
+
else if (score >= 0.6) {
|
|
106
|
+
return `●●○ ${(score * 100).toFixed(1)}%`;
|
|
107
|
+
}
|
|
108
|
+
else if (score >= 0.4) {
|
|
109
|
+
return `●○○ ${(score * 100).toFixed(1)}%`;
|
|
110
|
+
}
|
|
111
|
+
else {
|
|
112
|
+
return `○○○ ${(score * 100).toFixed(1)}%`;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
//# sourceMappingURL=search.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search.js","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AACjE,OAAO,EAAE,cAAc,EAAiB,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAiChE;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAAsB;IACtD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,GAAG,EAAE,EAAE,KAAK,GAAG,6CAA6C,EAAE,QAAQ,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC;IAEzH,uBAAuB;IACvB,MAAM,WAAW,GAAG,MAAM,kBAAkB,EAAE,CAAC;IAC/C,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,2FAA2F,CAAC,CAAC;IAC/G,CAAC;IAED,eAAe;IACf,MAAM,OAAO,GAAG,IAAI,cAAc,CAAC,WAAW,CAAC,CAAC;IAChD,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;IAE3B,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAEjC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO;YACL,KAAK;YACL,OAAO,EAAE,EAAE;YACX,UAAU,EAAE,CAAC;YACb,QAAQ,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI;SAC1C,CAAC;IACJ,CAAC;IAED,2BAA2B;IAC3B,MAAM,cAAc,GAAG,gBAAgB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IACrD,MAAM,cAAc,GAAG,MAAM,iBAAiB,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;IAEtE,yBAAyB;IACzB,MAAM,OAAO,GAAkD,EAAE,CAAC;IAElE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,qCAAqC;QACrC,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnD,SAAS;QACX,CAAC;QAED,oDAAoD;QACpD,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,cAAc,CAAC,MAAM,EAAE,CAAC;YACpD,OAAO,CAAC,IAAI,CAAC,qBAAqB,IAAI,CAAC,IAAI,2CAA2C,IAAI,CAAC,SAAS,CAAC,MAAM,OAAO,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC;YAC5I,SAAS;QACX,CAAC;QAED,MAAM,KAAK,GAAG,gBAAgB,CAAC,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAE/D,IAAI,KAAK,IAAI,QAAQ,EAAE,CAAC;YACtB,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,6BAA6B;IAC7B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAE1C,YAAY;IACZ,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAE1C,iBAAiB;IACjB,MAAM,gBAAgB,GAAmB,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC5E,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC/B,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,OAAO,EAAE,IAAI,CAAC,OAAO;KACtB,CAAC,CAAC,CAAC;IAEJ,MAAM,QAAQ,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC;IAEjD,OAAO;QACL,KAAK;QACL,OAAO,EAAE,gBAAgB;QACzB,UAAU,EAAE,OAAO,CAAC,MAAM;QAC1B,QAAQ;KACT,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,QAAwB;IAC1D,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,aAAa,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAC;IAC3C,KAAK,CAAC,IAAI,CAAC,SAAS,QAAQ,CAAC,UAAU,eAAe,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACvF,KAAK,CAAC,IAAI,CAAC,eAAe,QAAQ,CAAC,OAAO,CAAC,MAAM,aAAa,CAAC,CAAC;IAEhE,IAAI,QAAQ,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClC,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IACpC,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACjD,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACnC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QAEvE,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC,gBAAgB,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;QACtD,CAAC;QAED,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;YACpD,MAAM,OAAO,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5F,KAAK,CAAC,IAAI,CAAC,kBAAkB,OAAO,EAAE,CAAC,CAAC;QAC1C,CAAC;QAED,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5D,KAAK,CAAC,IAAI,CAAC,eAAe,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,KAAK,CAAC,IAAI,CAAC,SAAS,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QACxC,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,KAAa;IAChC,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;QACjB,OAAO,OAAO,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IAC5C,CAAC;SAAM,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;QACxB,OAAO,OAAO,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IAC5C,CAAC;SAAM,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;QACxB,OAAO,OAAO,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IAC5C,CAAC;SAAM,CAAC;QACN,OAAO,OAAO,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IAC5C,CAAC;AACH,CAAC"}
|