@mmnto/totem 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunkers/chunker.d.ts +12 -0
- package/dist/chunkers/chunker.d.ts.map +1 -0
- package/dist/chunkers/chunker.js +17 -0
- package/dist/chunkers/chunker.js.map +1 -0
- package/dist/chunkers/markdown-chunker.d.ts +17 -0
- package/dist/chunkers/markdown-chunker.d.ts.map +1 -0
- package/dist/chunkers/markdown-chunker.js +120 -0
- package/dist/chunkers/markdown-chunker.js.map +1 -0
- package/dist/chunkers/schema-file-chunker.d.ts +15 -0
- package/dist/chunkers/schema-file-chunker.d.ts.map +1 -0
- package/dist/chunkers/schema-file-chunker.js +55 -0
- package/dist/chunkers/schema-file-chunker.js.map +1 -0
- package/dist/chunkers/session-log-chunker.d.ts +14 -0
- package/dist/chunkers/session-log-chunker.d.ts.map +1 -0
- package/dist/chunkers/session-log-chunker.js +78 -0
- package/dist/chunkers/session-log-chunker.js.map +1 -0
- package/dist/chunkers/test-file-chunker.d.ts +17 -0
- package/dist/chunkers/test-file-chunker.d.ts.map +1 -0
- package/dist/chunkers/test-file-chunker.js +66 -0
- package/dist/chunkers/test-file-chunker.js.map +1 -0
- package/dist/chunkers/typescript-chunker.d.ts +17 -0
- package/dist/chunkers/typescript-chunker.d.ts.map +1 -0
- package/dist/chunkers/typescript-chunker.js +94 -0
- package/dist/chunkers/typescript-chunker.js.map +1 -0
- package/dist/config-schema.d.ts +229 -0
- package/dist/config-schema.d.ts.map +1 -0
- package/dist/config-schema.js +56 -0
- package/dist/config-schema.js.map +1 -0
- package/dist/embedders/embedder.d.ts +15 -0
- package/dist/embedders/embedder.d.ts.map +1 -0
- package/dist/embedders/embedder.js +11 -0
- package/dist/embedders/embedder.js.map +1 -0
- package/dist/embedders/ollama-embedder.d.ts +14 -0
- package/dist/embedders/ollama-embedder.d.ts.map +1 -0
- package/dist/embedders/ollama-embedder.js +67 -0
- package/dist/embedders/ollama-embedder.js.map +1 -0
- package/dist/embedders/openai-embedder.d.ts +9 -0
- package/dist/embedders/openai-embedder.d.ts.map +1 -0
- package/dist/embedders/openai-embedder.js +36 -0
- package/dist/embedders/openai-embedder.js.map +1 -0
- package/dist/index.d.ts +12 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -0
- package/dist/ingest/file-resolver.d.ts +14 -0
- package/dist/ingest/file-resolver.d.ts.map +1 -0
- package/dist/ingest/file-resolver.js +54 -0
- package/dist/ingest/file-resolver.js.map +1 -0
- package/dist/ingest/pipeline.d.ts +7 -0
- package/dist/ingest/pipeline.d.ts.map +1 -0
- package/dist/ingest/pipeline.js +79 -0
- package/dist/ingest/pipeline.js.map +1 -0
- package/dist/ingest/sync.d.ts +4 -0
- package/dist/ingest/sync.d.ts.map +1 -0
- package/dist/ingest/sync.js +3 -0
- package/dist/ingest/sync.js.map +1 -0
- package/dist/store/lance-schema.d.ts +42 -0
- package/dist/store/lance-schema.d.ts.map +1 -0
- package/dist/store/lance-schema.js +21 -0
- package/dist/store/lance-schema.js.map +1 -0
- package/dist/store/lance-store.d.ts +27 -0
- package/dist/store/lance-store.d.ts.map +1 -0
- package/dist/store/lance-store.js +106 -0
- package/dist/store/lance-store.js.map +1 -0
- package/dist/types.d.ts +75 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/package.json +52 -0
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
/**
|
|
3
|
+
* Zod schema for totem.config.ts — lives at the root of consuming projects.
|
|
4
|
+
*/
|
|
5
|
+
export declare const ChunkStrategySchema: z.ZodEnum<["typescript-ast", "markdown-heading", "session-log", "schema-file", "test-file"]>;
|
|
6
|
+
export declare const ContentTypeSchema: z.ZodEnum<["code", "session_log", "spec"]>;
|
|
7
|
+
export declare const IngestTargetSchema: z.ZodObject<{
|
|
8
|
+
glob: z.ZodString;
|
|
9
|
+
type: z.ZodEnum<["code", "session_log", "spec"]>;
|
|
10
|
+
strategy: z.ZodEnum<["typescript-ast", "markdown-heading", "session-log", "schema-file", "test-file"]>;
|
|
11
|
+
}, "strip", z.ZodTypeAny, {
|
|
12
|
+
glob: string;
|
|
13
|
+
type: "code" | "session_log" | "spec";
|
|
14
|
+
strategy: "typescript-ast" | "markdown-heading" | "session-log" | "schema-file" | "test-file";
|
|
15
|
+
}, {
|
|
16
|
+
glob: string;
|
|
17
|
+
type: "code" | "session_log" | "spec";
|
|
18
|
+
strategy: "typescript-ast" | "markdown-heading" | "session-log" | "schema-file" | "test-file";
|
|
19
|
+
}>;
|
|
20
|
+
export declare const OpenAIProviderSchema: z.ZodObject<{
|
|
21
|
+
provider: z.ZodLiteral<"openai">;
|
|
22
|
+
model: z.ZodDefault<z.ZodString>;
|
|
23
|
+
dimensions: z.ZodOptional<z.ZodNumber>;
|
|
24
|
+
}, "strip", z.ZodTypeAny, {
|
|
25
|
+
provider: "openai";
|
|
26
|
+
model: string;
|
|
27
|
+
dimensions?: number | undefined;
|
|
28
|
+
}, {
|
|
29
|
+
provider: "openai";
|
|
30
|
+
model?: string | undefined;
|
|
31
|
+
dimensions?: number | undefined;
|
|
32
|
+
}>;
|
|
33
|
+
export declare const OllamaProviderSchema: z.ZodObject<{
|
|
34
|
+
provider: z.ZodLiteral<"ollama">;
|
|
35
|
+
model: z.ZodDefault<z.ZodString>;
|
|
36
|
+
baseUrl: z.ZodDefault<z.ZodString>;
|
|
37
|
+
dimensions: z.ZodOptional<z.ZodNumber>;
|
|
38
|
+
}, "strip", z.ZodTypeAny, {
|
|
39
|
+
provider: "ollama";
|
|
40
|
+
model: string;
|
|
41
|
+
baseUrl: string;
|
|
42
|
+
dimensions?: number | undefined;
|
|
43
|
+
}, {
|
|
44
|
+
provider: "ollama";
|
|
45
|
+
model?: string | undefined;
|
|
46
|
+
dimensions?: number | undefined;
|
|
47
|
+
baseUrl?: string | undefined;
|
|
48
|
+
}>;
|
|
49
|
+
export declare const EmbeddingProviderSchema: z.ZodDiscriminatedUnion<"provider", [z.ZodObject<{
|
|
50
|
+
provider: z.ZodLiteral<"openai">;
|
|
51
|
+
model: z.ZodDefault<z.ZodString>;
|
|
52
|
+
dimensions: z.ZodOptional<z.ZodNumber>;
|
|
53
|
+
}, "strip", z.ZodTypeAny, {
|
|
54
|
+
provider: "openai";
|
|
55
|
+
model: string;
|
|
56
|
+
dimensions?: number | undefined;
|
|
57
|
+
}, {
|
|
58
|
+
provider: "openai";
|
|
59
|
+
model?: string | undefined;
|
|
60
|
+
dimensions?: number | undefined;
|
|
61
|
+
}>, z.ZodObject<{
|
|
62
|
+
provider: z.ZodLiteral<"ollama">;
|
|
63
|
+
model: z.ZodDefault<z.ZodString>;
|
|
64
|
+
baseUrl: z.ZodDefault<z.ZodString>;
|
|
65
|
+
dimensions: z.ZodOptional<z.ZodNumber>;
|
|
66
|
+
}, "strip", z.ZodTypeAny, {
|
|
67
|
+
provider: "ollama";
|
|
68
|
+
model: string;
|
|
69
|
+
baseUrl: string;
|
|
70
|
+
dimensions?: number | undefined;
|
|
71
|
+
}, {
|
|
72
|
+
provider: "ollama";
|
|
73
|
+
model?: string | undefined;
|
|
74
|
+
dimensions?: number | undefined;
|
|
75
|
+
baseUrl?: string | undefined;
|
|
76
|
+
}>]>;
|
|
77
|
+
export declare const DEFAULT_IGNORE_PATTERNS: string[];
|
|
78
|
+
export declare const ShellOrchestratorSchema: z.ZodObject<{
|
|
79
|
+
provider: z.ZodLiteral<"shell">;
|
|
80
|
+
/** Shell command with {file} and {model} placeholders */
|
|
81
|
+
command: z.ZodString;
|
|
82
|
+
/** Default model name substituted for {model} if --model is not passed */
|
|
83
|
+
defaultModel: z.ZodOptional<z.ZodString>;
|
|
84
|
+
}, "strip", z.ZodTypeAny, {
|
|
85
|
+
provider: "shell";
|
|
86
|
+
command: string;
|
|
87
|
+
defaultModel?: string | undefined;
|
|
88
|
+
}, {
|
|
89
|
+
provider: "shell";
|
|
90
|
+
command: string;
|
|
91
|
+
defaultModel?: string | undefined;
|
|
92
|
+
}>;
|
|
93
|
+
export declare const OrchestratorSchema: z.ZodDiscriminatedUnion<"provider", [z.ZodObject<{
|
|
94
|
+
provider: z.ZodLiteral<"shell">;
|
|
95
|
+
/** Shell command with {file} and {model} placeholders */
|
|
96
|
+
command: z.ZodString;
|
|
97
|
+
/** Default model name substituted for {model} if --model is not passed */
|
|
98
|
+
defaultModel: z.ZodOptional<z.ZodString>;
|
|
99
|
+
}, "strip", z.ZodTypeAny, {
|
|
100
|
+
provider: "shell";
|
|
101
|
+
command: string;
|
|
102
|
+
defaultModel?: string | undefined;
|
|
103
|
+
}, {
|
|
104
|
+
provider: "shell";
|
|
105
|
+
command: string;
|
|
106
|
+
defaultModel?: string | undefined;
|
|
107
|
+
}>]>;
|
|
108
|
+
export declare const TotemConfigSchema: z.ZodObject<{
|
|
109
|
+
/** Glob patterns and chunking strategies for each ingest target */
|
|
110
|
+
targets: z.ZodArray<z.ZodObject<{
|
|
111
|
+
glob: z.ZodString;
|
|
112
|
+
type: z.ZodEnum<["code", "session_log", "spec"]>;
|
|
113
|
+
strategy: z.ZodEnum<["typescript-ast", "markdown-heading", "session-log", "schema-file", "test-file"]>;
|
|
114
|
+
}, "strip", z.ZodTypeAny, {
|
|
115
|
+
glob: string;
|
|
116
|
+
type: "code" | "session_log" | "spec";
|
|
117
|
+
strategy: "typescript-ast" | "markdown-heading" | "session-log" | "schema-file" | "test-file";
|
|
118
|
+
}, {
|
|
119
|
+
glob: string;
|
|
120
|
+
type: "code" | "session_log" | "spec";
|
|
121
|
+
strategy: "typescript-ast" | "markdown-heading" | "session-log" | "schema-file" | "test-file";
|
|
122
|
+
}>, "many">;
|
|
123
|
+
/** Embedding provider configuration */
|
|
124
|
+
embedding: z.ZodDiscriminatedUnion<"provider", [z.ZodObject<{
|
|
125
|
+
provider: z.ZodLiteral<"openai">;
|
|
126
|
+
model: z.ZodDefault<z.ZodString>;
|
|
127
|
+
dimensions: z.ZodOptional<z.ZodNumber>;
|
|
128
|
+
}, "strip", z.ZodTypeAny, {
|
|
129
|
+
provider: "openai";
|
|
130
|
+
model: string;
|
|
131
|
+
dimensions?: number | undefined;
|
|
132
|
+
}, {
|
|
133
|
+
provider: "openai";
|
|
134
|
+
model?: string | undefined;
|
|
135
|
+
dimensions?: number | undefined;
|
|
136
|
+
}>, z.ZodObject<{
|
|
137
|
+
provider: z.ZodLiteral<"ollama">;
|
|
138
|
+
model: z.ZodDefault<z.ZodString>;
|
|
139
|
+
baseUrl: z.ZodDefault<z.ZodString>;
|
|
140
|
+
dimensions: z.ZodOptional<z.ZodNumber>;
|
|
141
|
+
}, "strip", z.ZodTypeAny, {
|
|
142
|
+
provider: "ollama";
|
|
143
|
+
model: string;
|
|
144
|
+
baseUrl: string;
|
|
145
|
+
dimensions?: number | undefined;
|
|
146
|
+
}, {
|
|
147
|
+
provider: "ollama";
|
|
148
|
+
model?: string | undefined;
|
|
149
|
+
dimensions?: number | undefined;
|
|
150
|
+
baseUrl?: string | undefined;
|
|
151
|
+
}>]>;
|
|
152
|
+
/** Optional: LLM orchestrator for spec/triage/shield commands */
|
|
153
|
+
orchestrator: z.ZodOptional<z.ZodDiscriminatedUnion<"provider", [z.ZodObject<{
|
|
154
|
+
provider: z.ZodLiteral<"shell">;
|
|
155
|
+
/** Shell command with {file} and {model} placeholders */
|
|
156
|
+
command: z.ZodString;
|
|
157
|
+
/** Default model name substituted for {model} if --model is not passed */
|
|
158
|
+
defaultModel: z.ZodOptional<z.ZodString>;
|
|
159
|
+
}, "strip", z.ZodTypeAny, {
|
|
160
|
+
provider: "shell";
|
|
161
|
+
command: string;
|
|
162
|
+
defaultModel?: string | undefined;
|
|
163
|
+
}, {
|
|
164
|
+
provider: "shell";
|
|
165
|
+
command: string;
|
|
166
|
+
defaultModel?: string | undefined;
|
|
167
|
+
}>]>>;
|
|
168
|
+
/** Optional: override the .totem/ directory path */
|
|
169
|
+
totemDir: z.ZodDefault<z.ZodString>;
|
|
170
|
+
/** Optional: override the .lancedb/ directory path */
|
|
171
|
+
lanceDir: z.ZodDefault<z.ZodString>;
|
|
172
|
+
/** Optional: glob patterns to exclude from indexing */
|
|
173
|
+
ignorePatterns: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
|
|
174
|
+
}, "strip", z.ZodTypeAny, {
|
|
175
|
+
targets: {
|
|
176
|
+
glob: string;
|
|
177
|
+
type: "code" | "session_log" | "spec";
|
|
178
|
+
strategy: "typescript-ast" | "markdown-heading" | "session-log" | "schema-file" | "test-file";
|
|
179
|
+
}[];
|
|
180
|
+
embedding: {
|
|
181
|
+
provider: "openai";
|
|
182
|
+
model: string;
|
|
183
|
+
dimensions?: number | undefined;
|
|
184
|
+
} | {
|
|
185
|
+
provider: "ollama";
|
|
186
|
+
model: string;
|
|
187
|
+
baseUrl: string;
|
|
188
|
+
dimensions?: number | undefined;
|
|
189
|
+
};
|
|
190
|
+
totemDir: string;
|
|
191
|
+
lanceDir: string;
|
|
192
|
+
ignorePatterns: string[];
|
|
193
|
+
orchestrator?: {
|
|
194
|
+
provider: "shell";
|
|
195
|
+
command: string;
|
|
196
|
+
defaultModel?: string | undefined;
|
|
197
|
+
} | undefined;
|
|
198
|
+
}, {
|
|
199
|
+
targets: {
|
|
200
|
+
glob: string;
|
|
201
|
+
type: "code" | "session_log" | "spec";
|
|
202
|
+
strategy: "typescript-ast" | "markdown-heading" | "session-log" | "schema-file" | "test-file";
|
|
203
|
+
}[];
|
|
204
|
+
embedding: {
|
|
205
|
+
provider: "openai";
|
|
206
|
+
model?: string | undefined;
|
|
207
|
+
dimensions?: number | undefined;
|
|
208
|
+
} | {
|
|
209
|
+
provider: "ollama";
|
|
210
|
+
model?: string | undefined;
|
|
211
|
+
dimensions?: number | undefined;
|
|
212
|
+
baseUrl?: string | undefined;
|
|
213
|
+
};
|
|
214
|
+
orchestrator?: {
|
|
215
|
+
provider: "shell";
|
|
216
|
+
command: string;
|
|
217
|
+
defaultModel?: string | undefined;
|
|
218
|
+
} | undefined;
|
|
219
|
+
totemDir?: string | undefined;
|
|
220
|
+
lanceDir?: string | undefined;
|
|
221
|
+
ignorePatterns?: string[] | undefined;
|
|
222
|
+
}>;
|
|
223
|
+
export type ChunkStrategy = z.infer<typeof ChunkStrategySchema>;
|
|
224
|
+
export type ContentType = z.infer<typeof ContentTypeSchema>;
|
|
225
|
+
export type IngestTarget = z.infer<typeof IngestTargetSchema>;
|
|
226
|
+
export type EmbeddingProvider = z.infer<typeof EmbeddingProviderSchema>;
|
|
227
|
+
export type Orchestrator = z.infer<typeof OrchestratorSchema>;
|
|
228
|
+
export type TotemConfig = z.infer<typeof TotemConfigSchema>;
|
|
229
|
+
//# sourceMappingURL=config-schema.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config-schema.d.ts","sourceRoot":"","sources":["../src/config-schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB;;GAEG;AAEH,eAAO,MAAM,mBAAmB,8FAM9B,CAAC;AAEH,eAAO,MAAM,iBAAiB,4CAA0C,CAAC;AAEzE,eAAO,MAAM,kBAAkB;;;;;;;;;;;;EAI7B,CAAC;AAEH,eAAO,MAAM,oBAAoB;;;;;;;;;;;;EAI/B,CAAC;AAEH,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;EAK/B,CAAC;AAEH,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;;;IAGlC,CAAC;AAEH,eAAO,MAAM,uBAAuB,UAAyD,CAAC;AAE9F,eAAO,MAAM,uBAAuB;;IAElC,yDAAyD;;IAEzD,0EAA0E;;;;;;;;;;EAE1E,CAAC;AAEH,eAAO,MAAM,kBAAkB;;IAN7B,yDAAyD;;IAEzD,0EAA0E;;;;;;;;;;IAIiB,CAAC;AAE9F,eAAO,MAAM,iBAAiB;IAC5B,mEAAmE;;;;;;;;;;;;;;IAGnE,uCAAuC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAGvC,iEAAiE;;;QAfjE,yDAAyD;;QAEzD,0EAA0E;;;;;;;;;;;IAgB1E,oDAAoD;;IAGpD,sDAAsD;;IAGtD,uDAAuD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAEvD,CAAC;AAEH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAChE,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAC9D,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AACxE,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAC9D,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC"}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
/**
|
|
3
|
+
* Zod schema for totem.config.ts — lives at the root of consuming projects.
|
|
4
|
+
*/
|
|
5
|
+
export const ChunkStrategySchema = z.enum([
|
|
6
|
+
'typescript-ast',
|
|
7
|
+
'markdown-heading',
|
|
8
|
+
'session-log',
|
|
9
|
+
'schema-file',
|
|
10
|
+
'test-file',
|
|
11
|
+
]);
|
|
12
|
+
export const ContentTypeSchema = z.enum(['code', 'session_log', 'spec']);
|
|
13
|
+
export const IngestTargetSchema = z.object({
|
|
14
|
+
glob: z.string(),
|
|
15
|
+
type: ContentTypeSchema,
|
|
16
|
+
strategy: ChunkStrategySchema,
|
|
17
|
+
});
|
|
18
|
+
export const OpenAIProviderSchema = z.object({
|
|
19
|
+
provider: z.literal('openai'),
|
|
20
|
+
model: z.string().default('text-embedding-3-small'),
|
|
21
|
+
dimensions: z.number().int().positive().optional(),
|
|
22
|
+
});
|
|
23
|
+
export const OllamaProviderSchema = z.object({
|
|
24
|
+
provider: z.literal('ollama'),
|
|
25
|
+
model: z.string().default('nomic-embed-text'),
|
|
26
|
+
baseUrl: z.string().default('http://localhost:11434'),
|
|
27
|
+
dimensions: z.number().int().positive().optional(),
|
|
28
|
+
});
|
|
29
|
+
export const EmbeddingProviderSchema = z.discriminatedUnion('provider', [
|
|
30
|
+
OpenAIProviderSchema,
|
|
31
|
+
OllamaProviderSchema,
|
|
32
|
+
]);
|
|
33
|
+
export const DEFAULT_IGNORE_PATTERNS = ['**/node_modules/**', '**/.lancedb/**', '**/dist/**'];
|
|
34
|
+
export const ShellOrchestratorSchema = z.object({
|
|
35
|
+
provider: z.literal('shell'),
|
|
36
|
+
/** Shell command with {file} and {model} placeholders */
|
|
37
|
+
command: z.string(),
|
|
38
|
+
/** Default model name substituted for {model} if --model is not passed */
|
|
39
|
+
defaultModel: z.string().optional(),
|
|
40
|
+
});
|
|
41
|
+
export const OrchestratorSchema = z.discriminatedUnion('provider', [ShellOrchestratorSchema]);
|
|
42
|
+
export const TotemConfigSchema = z.object({
|
|
43
|
+
/** Glob patterns and chunking strategies for each ingest target */
|
|
44
|
+
targets: z.array(IngestTargetSchema).min(1),
|
|
45
|
+
/** Embedding provider configuration */
|
|
46
|
+
embedding: EmbeddingProviderSchema,
|
|
47
|
+
/** Optional: LLM orchestrator for spec/triage/shield commands */
|
|
48
|
+
orchestrator: OrchestratorSchema.optional(),
|
|
49
|
+
/** Optional: override the .totem/ directory path */
|
|
50
|
+
totemDir: z.string().default('.totem'),
|
|
51
|
+
/** Optional: override the .lancedb/ directory path */
|
|
52
|
+
lanceDir: z.string().default('.lancedb'),
|
|
53
|
+
/** Optional: glob patterns to exclude from indexing */
|
|
54
|
+
ignorePatterns: z.array(z.string()).default(DEFAULT_IGNORE_PATTERNS),
|
|
55
|
+
});
|
|
56
|
+
//# sourceMappingURL=config-schema.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config-schema.js","sourceRoot":"","sources":["../src/config-schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB;;GAEG;AAEH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC,IAAI,CAAC;IACxC,gBAAgB;IAChB,kBAAkB;IAClB,aAAa;IACb,aAAa;IACb,WAAW;CACZ,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC;AAEzE,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IACzC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,IAAI,EAAE,iBAAiB;IACvB,QAAQ,EAAE,mBAAmB;CAC9B,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3C,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC;IAC7B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,wBAAwB,CAAC;IACnD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CACnD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3C,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC;IAC7B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,kBAAkB,CAAC;IAC7C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,wBAAwB,CAAC;IACrD,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CACnD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,kBAAkB,CAAC,UAAU,EAAE;IACtE,oBAAoB;IACpB,oBAAoB;CACrB,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,oBAAoB,EAAE,gBAAgB,EAAE,YAAY,CAAC,CAAC;AAE9F,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9C,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;IAC5B,yDAAyD;IACzD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,0EAA0E;IAC1E,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACpC,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,kBAAkB,CAAC,UAAU,EAAE,CAAC,uBAAuB,CAAC,CAAC,CAAC;AAE9F,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,mEAAmE;IACnE,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAE3C,uCAAuC;IACvC,SAAS,EAAE,uBAAuB;IAElC,iEAAiE;IACjE,YAAY,EAAE,kBAAkB,CAAC,QAAQ,EAAE;IAE3C,oDAAoD;IACpD,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC;IAEtC,sDAAsD;IACtD,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC;IAExC,uDAAuD;IACvD,cAAc,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,uBAAuB,CAAC;CACrE,CAAC,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { EmbeddingProvider } from '../config-schema.js';
|
|
2
|
+
/**
|
|
3
|
+
* All embedding providers implement this interface.
|
|
4
|
+
*/
|
|
5
|
+
export interface Embedder {
|
|
6
|
+
/** The dimensionality of vectors produced by this embedder */
|
|
7
|
+
readonly dimensions: number;
|
|
8
|
+
/**
|
|
9
|
+
* Embed a batch of texts. Returns one vector per input text.
|
|
10
|
+
* Implementations should handle batching/rate-limiting internally.
|
|
11
|
+
*/
|
|
12
|
+
embed(texts: string[]): Promise<number[][]>;
|
|
13
|
+
}
|
|
14
|
+
export declare function createEmbedder(config: EmbeddingProvider): Embedder;
|
|
15
|
+
//# sourceMappingURL=embedder.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../../src/embedders/embedder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAI7D;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,8DAA8D;IAC9D,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAE5B;;;OAGG;IACH,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;CAC7C;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,iBAAiB,GAAG,QAAQ,CAOlE"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { OllamaEmbedder } from './ollama-embedder.js';
|
|
2
|
+
import { OpenAIEmbedder } from './openai-embedder.js';
|
|
3
|
+
export function createEmbedder(config) {
|
|
4
|
+
switch (config.provider) {
|
|
5
|
+
case 'openai':
|
|
6
|
+
return new OpenAIEmbedder(config.model, config.dimensions);
|
|
7
|
+
case 'ollama':
|
|
8
|
+
return new OllamaEmbedder(config.model, config.baseUrl, config.dimensions);
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=embedder.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../../src/embedders/embedder.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAgBtD,MAAM,UAAU,cAAc,CAAC,MAAyB;IACtD,QAAQ,MAAM,CAAC,QAAQ,EAAE,CAAC;QACxB,KAAK,QAAQ;YACX,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;QAC7D,KAAK,QAAQ;YACX,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IAC/E,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { Embedder } from './embedder.js';
|
|
2
|
+
/**
|
|
3
|
+
* Ollama embedding via the /api/embed endpoint.
|
|
4
|
+
* No SDK required — plain fetch.
|
|
5
|
+
*/
|
|
6
|
+
export declare class OllamaEmbedder implements Embedder {
|
|
7
|
+
readonly dimensions: number;
|
|
8
|
+
private model;
|
|
9
|
+
private baseUrl;
|
|
10
|
+
constructor(model?: string, baseUrl?: string, dimensions?: number);
|
|
11
|
+
private embedBatch;
|
|
12
|
+
embed(texts: string[]): Promise<number[][]>;
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=ollama-embedder.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ollama-embedder.d.ts","sourceRoot":"","sources":["../../src/embedders/ollama-embedder.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAM9C;;;GAGG;AACH,qBAAa,cAAe,YAAW,QAAQ;IAC7C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,OAAO,CAAS;gBAGtB,KAAK,GAAE,MAA2B,EAClC,OAAO,GAAE,MAAiC,EAC1C,UAAU,CAAC,EAAE,MAAM;YAOP,UAAU;IAqBlB,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CAoClD"}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
const DEFAULT_DIMENSIONS = 768;
|
|
3
|
+
const MAX_BATCH_SIZE = 10;
|
|
4
|
+
const MAX_CHARS_PER_TEXT = 6_000; // ~2000 tokens, conservative for nomic-embed-text's 8192 context
|
|
5
|
+
/**
|
|
6
|
+
* Ollama embedding via the /api/embed endpoint.
|
|
7
|
+
* No SDK required — plain fetch.
|
|
8
|
+
*/
|
|
9
|
+
export class OllamaEmbedder {
|
|
10
|
+
dimensions;
|
|
11
|
+
model;
|
|
12
|
+
baseUrl;
|
|
13
|
+
constructor(model = 'nomic-embed-text', baseUrl = 'http://localhost:11434', dimensions) {
|
|
14
|
+
this.model = model;
|
|
15
|
+
this.baseUrl = baseUrl;
|
|
16
|
+
this.dimensions = dimensions ?? DEFAULT_DIMENSIONS;
|
|
17
|
+
}
|
|
18
|
+
async embedBatch(texts) {
|
|
19
|
+
const response = await fetch(`${this.baseUrl}/api/embed`, {
|
|
20
|
+
method: 'POST',
|
|
21
|
+
headers: { 'Content-Type': 'application/json' },
|
|
22
|
+
body: JSON.stringify({
|
|
23
|
+
model: this.model,
|
|
24
|
+
input: texts,
|
|
25
|
+
}),
|
|
26
|
+
});
|
|
27
|
+
if (!response.ok) {
|
|
28
|
+
const body = await response.text();
|
|
29
|
+
throw new Error(`[Totem Error] Ollama embedding failed (${response.status}): ${body}`);
|
|
30
|
+
}
|
|
31
|
+
const data = z
|
|
32
|
+
.object({ embeddings: z.array(z.array(z.number())) })
|
|
33
|
+
.parse(await response.json());
|
|
34
|
+
return data.embeddings;
|
|
35
|
+
}
|
|
36
|
+
async embed(texts) {
|
|
37
|
+
if (texts.length === 0)
|
|
38
|
+
return [];
|
|
39
|
+
// Truncate oversized texts to stay within model context window
|
|
40
|
+
const truncated = texts.map((t) => t.length > MAX_CHARS_PER_TEXT ? t.slice(0, MAX_CHARS_PER_TEXT) : t);
|
|
41
|
+
const results = [];
|
|
42
|
+
// Batch to avoid overwhelming Ollama with large payloads
|
|
43
|
+
for (let i = 0; i < truncated.length; i += MAX_BATCH_SIZE) {
|
|
44
|
+
const batch = truncated.slice(i, i + MAX_BATCH_SIZE);
|
|
45
|
+
try {
|
|
46
|
+
const embeddings = await this.embedBatch(batch);
|
|
47
|
+
results.push(...embeddings);
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
// Batch failed — retry each text individually
|
|
51
|
+
for (const text of batch) {
|
|
52
|
+
try {
|
|
53
|
+
const [embedding] = await this.embedBatch([text]);
|
|
54
|
+
results.push(embedding);
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
// Individual text exceeds context — use zero vector
|
|
58
|
+
console.error(`[Totem] Skipping oversized chunk (${text.length} chars): ${text.slice(0, 60)}...`);
|
|
59
|
+
results.push(new Array(this.dimensions).fill(0));
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return results;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
//# sourceMappingURL=ollama-embedder.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ollama-embedder.js","sourceRoot":"","sources":["../../src/embedders/ollama-embedder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAC/B,MAAM,cAAc,GAAG,EAAE,CAAC;AAC1B,MAAM,kBAAkB,GAAG,KAAK,CAAC,CAAC,iEAAiE;AAEnG;;;GAGG;AACH,MAAM,OAAO,cAAc;IAChB,UAAU,CAAS;IACpB,KAAK,CAAS;IACd,OAAO,CAAS;IAExB,YACE,QAAgB,kBAAkB,EAClC,UAAkB,wBAAwB,EAC1C,UAAmB;QAEnB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,UAAU,GAAG,UAAU,IAAI,kBAAkB,CAAC;IACrD,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,KAAe;QACtC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,YAAY,EAAE;YACxD,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,KAAK,EAAE,KAAK;aACb,CAAC;SACH,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,0CAA0C,QAAQ,CAAC,MAAM,MAAM,IAAI,EAAE,CAAC,CAAC;QACzF,CAAC;QAED,MAAM,IAAI,GAAG,CAAC;aACX,MAAM,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC;aACpD,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QAChC,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAe;QACzB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAElC,+DAA+D;QAC/D,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAChC,CAAC,CAAC,MAAM,GAAG,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,CACnE,CAAC;QAEF,MAAM,OAAO,GAAe,EAAE,CAAC;QAE/B,yDAAyD;QACzD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,cAAc,EAAE,CAAC;YAC1D,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,CAAC;YAErD,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;gBAChD,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC;YAC9B,CAAC;YAAC,MAAM,CAAC;gBACP,8CAA8C;gBAC9C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBACzB,IAAI,CAAC;wBACH,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;wBAClD,OAAO,CAAC,IAAI,CAAC,SAAU,CAAC,CAAC;oBAC3B,CAAC;oBAAC,MAAM,CAAC;wBACP,oDAAoD;wBACpD,OAAO,CAAC,KAAK,CACX,qCAAqC,IAAI,CAAC,MAAM,YAAY,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CACnF,CAAC;wBACF,OAAO,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBACnD,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { Embedder } from './embedder.js';
|
|
2
|
+
export declare class OpenAIEmbedder implements Embedder {
|
|
3
|
+
readonly dimensions: number;
|
|
4
|
+
private client;
|
|
5
|
+
private model;
|
|
6
|
+
constructor(model?: string, dimensions?: number);
|
|
7
|
+
embed(texts: string[]): Promise<number[][]>;
|
|
8
|
+
}
|
|
9
|
+
//# sourceMappingURL=openai-embedder.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"openai-embedder.d.ts","sourceRoot":"","sources":["../../src/embedders/openai-embedder.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAK9C,qBAAa,cAAe,YAAW,QAAQ;IAC7C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAS;gBAEV,KAAK,GAAE,MAAiC,EAAE,UAAU,CAAC,EAAE,MAAM;IAenE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CAoBlD"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import OpenAI from 'openai';
|
|
2
|
+
const MAX_BATCH_SIZE = 2048;
|
|
3
|
+
const DEFAULT_DIMENSIONS = 1536;
|
|
4
|
+
export class OpenAIEmbedder {
|
|
5
|
+
dimensions;
|
|
6
|
+
client;
|
|
7
|
+
model;
|
|
8
|
+
constructor(model = 'text-embedding-3-small', dimensions) {
|
|
9
|
+
this.model = model;
|
|
10
|
+
this.dimensions = dimensions ?? DEFAULT_DIMENSIONS;
|
|
11
|
+
const apiKey = process.env['OPENAI_API_KEY'];
|
|
12
|
+
if (!apiKey) {
|
|
13
|
+
throw new Error('[Totem Error] No embedding provider configured.\n' +
|
|
14
|
+
"Set OPENAI_API_KEY in your .env or configure 'ollama' in totem.config.ts.");
|
|
15
|
+
}
|
|
16
|
+
this.client = new OpenAI({ apiKey });
|
|
17
|
+
}
|
|
18
|
+
async embed(texts) {
|
|
19
|
+
if (texts.length === 0)
|
|
20
|
+
return [];
|
|
21
|
+
const results = [];
|
|
22
|
+
for (let i = 0; i < texts.length; i += MAX_BATCH_SIZE) {
|
|
23
|
+
const batch = texts.slice(i, i + MAX_BATCH_SIZE);
|
|
24
|
+
const response = await this.client.embeddings.create({
|
|
25
|
+
model: this.model,
|
|
26
|
+
input: batch,
|
|
27
|
+
});
|
|
28
|
+
const sorted = response.data.sort((a, b) => a.index - b.index);
|
|
29
|
+
for (const item of sorted) {
|
|
30
|
+
results.push(item.embedding);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
return results;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
//# sourceMappingURL=openai-embedder.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"openai-embedder.js","sourceRoot":"","sources":["../../src/embedders/openai-embedder.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAI5B,MAAM,cAAc,GAAG,IAAI,CAAC;AAC5B,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAEhC,MAAM,OAAO,cAAc;IAChB,UAAU,CAAS;IACpB,MAAM,CAAS;IACf,KAAK,CAAS;IAEtB,YAAY,QAAgB,wBAAwB,EAAE,UAAmB;QACvE,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,UAAU,GAAG,UAAU,IAAI,kBAAkB,CAAC;QAEnD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QAC7C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CACb,mDAAmD;gBACjD,2EAA2E,CAC9E,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAe;QACzB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAElC,MAAM,OAAO,GAAe,EAAE,CAAC;QAE/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,cAAc,EAAE,CAAC;YACtD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,CAAC;YACjD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;gBACnD,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,KAAK,EAAE,KAAK;aACb,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;YAC/D,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;gBAC1B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export type { ChunkStrategy, ContentType, EmbeddingProvider, IngestTarget, Orchestrator, TotemConfig, } from './config-schema.js';
|
|
2
|
+
export { ChunkStrategySchema, ContentTypeSchema, DEFAULT_IGNORE_PATTERNS, EmbeddingProviderSchema, IngestTargetSchema, OllamaProviderSchema, OpenAIProviderSchema, OrchestratorSchema, ShellOrchestratorSchema, TotemConfigSchema, } from './config-schema.js';
|
|
3
|
+
export type { Chunk, SearchOptions, SearchResult, StoredChunk, SyncOptions } from './types.js';
|
|
4
|
+
export type { Chunker } from './chunkers/chunker.js';
|
|
5
|
+
export { createChunker } from './chunkers/chunker.js';
|
|
6
|
+
export type { Embedder } from './embedders/embedder.js';
|
|
7
|
+
export { createEmbedder } from './embedders/embedder.js';
|
|
8
|
+
export { TOTEM_TABLE_NAME } from './store/lance-schema.js';
|
|
9
|
+
export { LanceStore } from './store/lance-store.js';
|
|
10
|
+
export type { ResolvedFile } from './ingest/sync.js';
|
|
11
|
+
export { getChangedFiles, resolveFiles, runSync } from './ingest/sync.js';
|
|
12
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,YAAY,EACV,aAAa,EACb,WAAW,EACX,iBAAiB,EACjB,YAAY,EACZ,YAAY,EACZ,WAAW,GACZ,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,mBAAmB,EACnB,iBAAiB,EACjB,uBAAuB,EACvB,uBAAuB,EACvB,kBAAkB,EAClB,oBAAoB,EACpB,oBAAoB,EACpB,kBAAkB,EAClB,uBAAuB,EACvB,iBAAiB,GAClB,MAAM,oBAAoB,CAAC;AAG5B,YAAY,EAAE,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAG/F,YAAY,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAGtD,YAAY,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAGzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAGpD,YAAY,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export { ChunkStrategySchema, ContentTypeSchema, DEFAULT_IGNORE_PATTERNS, EmbeddingProviderSchema, IngestTargetSchema, OllamaProviderSchema, OpenAIProviderSchema, OrchestratorSchema, ShellOrchestratorSchema, TotemConfigSchema, } from './config-schema.js';
|
|
2
|
+
export { createChunker } from './chunkers/chunker.js';
|
|
3
|
+
export { createEmbedder } from './embedders/embedder.js';
|
|
4
|
+
// Store
|
|
5
|
+
export { TOTEM_TABLE_NAME } from './store/lance-schema.js';
|
|
6
|
+
export { LanceStore } from './store/lance-store.js';
|
|
7
|
+
export { getChangedFiles, resolveFiles, runSync } from './ingest/sync.js';
|
|
8
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AASA,OAAO,EACL,mBAAmB,EACnB,iBAAiB,EACjB,uBAAuB,EACvB,uBAAuB,EACvB,kBAAkB,EAClB,oBAAoB,EACpB,oBAAoB,EACpB,kBAAkB,EAClB,uBAAuB,EACvB,iBAAiB,GAClB,MAAM,oBAAoB,CAAC;AAO5B,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAItD,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAEzD,QAAQ;AACR,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAIpD,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { IngestTarget } from '../config-schema.js';
|
|
2
|
+
export interface ResolvedFile {
|
|
3
|
+
absolutePath: string;
|
|
4
|
+
relativePath: string;
|
|
5
|
+
target: IngestTarget;
|
|
6
|
+
}
|
|
7
|
+
/** Resolve glob patterns from config targets to actual file paths. */
|
|
8
|
+
export declare function resolveFiles(targets: IngestTarget[], projectRoot: string, ignorePatterns?: string[]): ResolvedFile[];
|
|
9
|
+
/**
|
|
10
|
+
* Get files changed since a given git ref (e.g., HEAD~1 or a commit SHA).
|
|
11
|
+
* Used for incremental sync.
|
|
12
|
+
*/
|
|
13
|
+
export declare function getChangedFiles(projectRoot: string, sinceRef?: string, onWarn?: (msg: string) => void): string[] | null;
|
|
14
|
+
//# sourceMappingURL=file-resolver.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"file-resolver.d.ts","sourceRoot":"","sources":["../../src/ingest/file-resolver.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAGxD,MAAM,WAAW,YAAY;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,YAAY,CAAC;CACtB;AAED,sEAAsE;AACtE,wBAAgB,YAAY,CAC1B,OAAO,EAAE,YAAY,EAAE,EACvB,WAAW,EAAE,MAAM,EACnB,cAAc,GAAE,MAAM,EAA4B,GACjD,YAAY,EAAE,CAwBhB;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAC7B,WAAW,EAAE,MAAM,EACnB,QAAQ,GAAE,MAAiB,EAC3B,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,GAC7B,MAAM,EAAE,GAAG,IAAI,CAmBjB"}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { execSync } from 'node:child_process';
|
|
2
|
+
import * as path from 'node:path';
|
|
3
|
+
import { globSync } from 'glob';
|
|
4
|
+
import { DEFAULT_IGNORE_PATTERNS } from '../config-schema.js';
|
|
5
|
+
/** Resolve glob patterns from config targets to actual file paths. */
|
|
6
|
+
export function resolveFiles(targets, projectRoot, ignorePatterns = DEFAULT_IGNORE_PATTERNS) {
|
|
7
|
+
const results = [];
|
|
8
|
+
const seen = new Set();
|
|
9
|
+
for (const target of targets) {
|
|
10
|
+
const matches = globSync(target.glob, {
|
|
11
|
+
cwd: projectRoot,
|
|
12
|
+
nodir: true,
|
|
13
|
+
ignore: ignorePatterns,
|
|
14
|
+
});
|
|
15
|
+
for (const relativePath of matches) {
|
|
16
|
+
if (seen.has(relativePath))
|
|
17
|
+
continue;
|
|
18
|
+
seen.add(relativePath);
|
|
19
|
+
results.push({
|
|
20
|
+
absolutePath: path.join(projectRoot, relativePath),
|
|
21
|
+
relativePath,
|
|
22
|
+
target,
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
return results;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Get files changed since a given git ref (e.g., HEAD~1 or a commit SHA).
|
|
30
|
+
* Used for incremental sync.
|
|
31
|
+
*/
|
|
32
|
+
export function getChangedFiles(projectRoot, sinceRef = 'HEAD~1', onWarn) {
|
|
33
|
+
try {
|
|
34
|
+
const output = execSync(`git diff --name-only ${sinceRef}`, {
|
|
35
|
+
cwd: projectRoot,
|
|
36
|
+
encoding: 'utf-8',
|
|
37
|
+
});
|
|
38
|
+
return output
|
|
39
|
+
.split('\n')
|
|
40
|
+
.map((line) => line.trim())
|
|
41
|
+
.filter(Boolean);
|
|
42
|
+
}
|
|
43
|
+
catch (err) {
|
|
44
|
+
const msg = `Failed to get changed files from git. Error: ${err instanceof Error ? err.message : String(err)}`;
|
|
45
|
+
if (onWarn) {
|
|
46
|
+
onWarn(msg);
|
|
47
|
+
}
|
|
48
|
+
else {
|
|
49
|
+
console.warn(`[Totem Warning] ${msg}`);
|
|
50
|
+
}
|
|
51
|
+
return null;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
//# sourceMappingURL=file-resolver.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"file-resolver.js","sourceRoot":"","sources":["../../src/ingest/file-resolver.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAGhC,OAAO,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAQ9D,sEAAsE;AACtE,MAAM,UAAU,YAAY,CAC1B,OAAuB,EACvB,WAAmB,EACnB,iBAA2B,uBAAuB;IAElD,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE;YACpC,GAAG,EAAE,WAAW;YAChB,KAAK,EAAE,IAAI;YACX,MAAM,EAAE,cAAc;SACvB,CAAC,CAAC;QAEH,KAAK,MAAM,YAAY,IAAI,OAAO,EAAE,CAAC;YACnC,IAAI,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC;gBAAE,SAAS;YACrC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;YAEvB,OAAO,CAAC,IAAI,CAAC;gBACX,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,YAAY,CAAC;gBAClD,YAAY;gBACZ,MAAM;aACP,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAC7B,WAAmB,EACnB,WAAmB,QAAQ,EAC3B,MAA8B;IAE9B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,QAAQ,CAAC,wBAAwB,QAAQ,EAAE,EAAE;YAC1D,GAAG,EAAE,WAAW;YAChB,QAAQ,EAAE,OAAO;SAClB,CAAC,CAAC;QACH,OAAO,MAAM;aACV,KAAK,CAAC,IAAI,CAAC;aACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;aAC1B,MAAM,CAAC,OAAO,CAAC,CAAC;IACrB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,gDAAgD,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/G,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,CAAC,GAAG,CAAC,CAAC;QACd,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CAAC,mBAAmB,GAAG,EAAE,CAAC,CAAC;QACzC,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { TotemConfig } from '../config-schema.js';
|
|
2
|
+
import type { SyncOptions } from '../types.js';
|
|
3
|
+
export declare function runSync(config: TotemConfig, options: SyncOptions): Promise<{
|
|
4
|
+
chunksProcessed: number;
|
|
5
|
+
filesProcessed: number;
|
|
6
|
+
}>;
|
|
7
|
+
//# sourceMappingURL=pipeline.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/ingest/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAGvD,OAAO,KAAK,EAAS,WAAW,EAAE,MAAM,aAAa,CAAC;AAMtD,wBAAsB,OAAO,CAC3B,MAAM,EAAE,WAAW,EACnB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC;IAAE,eAAe,EAAE,MAAM,CAAC;IAAC,cAAc,EAAE,MAAM,CAAA;CAAE,CAAC,CAoF9D"}
|