@sparkleideas/embeddings 3.0.0-alpha.15 → 3.0.0-alpha.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/embedding-service.ts +25 -25
- package/src/index.ts +1 -1
- package/src/neural-integration.ts +8 -8
- package/src/types.ts +3 -3
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sparkleideas/embeddings",
|
|
3
|
-
"version": "3.0.0-alpha.
|
|
3
|
+
"version": "3.0.0-alpha.16",
|
|
4
4
|
"description": "V3 Embedding Service - OpenAI, Transformers.js, Agentic-Flow (ONNX), Mock providers with hyperbolic embeddings, normalization, and chunking",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
package/src/embedding-service.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* V3 Embedding Service Implementation
|
|
3
3
|
*
|
|
4
|
-
* Production embedding service aligned with
|
|
4
|
+
* Production embedding service aligned with agentic-flow@alpha:
|
|
5
5
|
* - OpenAI provider (text-embedding-3-small/large)
|
|
6
6
|
* - Transformers.js provider (local ONNX models)
|
|
7
7
|
* - Mock provider (development/testing)
|
|
@@ -598,7 +598,7 @@ export class MockEmbeddingService extends BaseEmbeddingService {
|
|
|
598
598
|
* - 3-4x faster batch processing
|
|
599
599
|
*/
|
|
600
600
|
export class AgenticFlowEmbeddingService extends BaseEmbeddingService {
|
|
601
|
-
readonly provider: EmbeddingProvider = '
|
|
601
|
+
readonly provider: EmbeddingProvider = 'agentic-flow';
|
|
602
602
|
private embedder: any = null;
|
|
603
603
|
private initialized = false;
|
|
604
604
|
private readonly modelId: string;
|
|
@@ -656,7 +656,7 @@ export class AgenticFlowEmbeddingService extends BaseEmbeddingService {
|
|
|
656
656
|
const possiblePaths: string[] = [];
|
|
657
657
|
|
|
658
658
|
// Try proper package exports first (preferred)
|
|
659
|
-
possiblePaths.push('
|
|
659
|
+
possiblePaths.push('agentic-flow/embeddings');
|
|
660
660
|
|
|
661
661
|
// Try node_modules resolution from different locations (for file:// imports)
|
|
662
662
|
try {
|
|
@@ -689,9 +689,9 @@ export class AgenticFlowEmbeddingService extends BaseEmbeddingService {
|
|
|
689
689
|
|
|
690
690
|
const errorDetail = lastError?.message ? ` Last error: ${lastError.message}` : '';
|
|
691
691
|
throw new Error(
|
|
692
|
-
`Failed to initialize
|
|
693
|
-
`Ensure
|
|
694
|
-
`npx
|
|
692
|
+
`Failed to initialize agentic-flow embeddings.${errorDetail} ` +
|
|
693
|
+
`Ensure agentic-flow is installed and ONNX model is downloaded: ` +
|
|
694
|
+
`npx agentic-flow@alpha embeddings init`
|
|
695
695
|
);
|
|
696
696
|
}
|
|
697
697
|
|
|
@@ -713,7 +713,7 @@ export class AgenticFlowEmbeddingService extends BaseEmbeddingService {
|
|
|
713
713
|
const startTime = performance.now();
|
|
714
714
|
|
|
715
715
|
try {
|
|
716
|
-
// Use
|
|
716
|
+
// Use agentic-flow's optimized embedder (has its own internal cache)
|
|
717
717
|
const embedding = await this.embedder.embed(text);
|
|
718
718
|
|
|
719
719
|
// Store in our cache as well
|
|
@@ -820,11 +820,11 @@ export class AgenticFlowEmbeddingService extends BaseEmbeddingService {
|
|
|
820
820
|
// ============================================================================
|
|
821
821
|
|
|
822
822
|
/**
|
|
823
|
-
* Check if
|
|
823
|
+
* Check if agentic-flow is available
|
|
824
824
|
*/
|
|
825
825
|
async function isAgenticFlowAvailable(): Promise<boolean> {
|
|
826
826
|
try {
|
|
827
|
-
await import('
|
|
827
|
+
await import('agentic-flow/embeddings');
|
|
828
828
|
return true;
|
|
829
829
|
} catch {
|
|
830
830
|
return false;
|
|
@@ -832,7 +832,7 @@ async function isAgenticFlowAvailable(): Promise<boolean> {
|
|
|
832
832
|
}
|
|
833
833
|
|
|
834
834
|
/**
|
|
835
|
-
* Auto-install
|
|
835
|
+
* Auto-install agentic-flow and initialize model
|
|
836
836
|
*/
|
|
837
837
|
async function autoInstallAgenticFlow(): Promise<boolean> {
|
|
838
838
|
const { exec } = await import('child_process');
|
|
@@ -845,12 +845,12 @@ async function autoInstallAgenticFlow(): Promise<boolean> {
|
|
|
845
845
|
return true;
|
|
846
846
|
}
|
|
847
847
|
|
|
848
|
-
console.log('[embeddings] Installing
|
|
849
|
-
await execAsync('npm install
|
|
848
|
+
console.log('[embeddings] Installing agentic-flow@alpha...');
|
|
849
|
+
await execAsync('npm install agentic-flow@alpha --save', { timeout: 120000 });
|
|
850
850
|
|
|
851
851
|
// Initialize the model
|
|
852
852
|
console.log('[embeddings] Downloading embedding model...');
|
|
853
|
-
await execAsync('npx
|
|
853
|
+
await execAsync('npx agentic-flow@alpha embeddings init', { timeout: 300000 });
|
|
854
854
|
|
|
855
855
|
// Verify installation
|
|
856
856
|
return await isAgenticFlowAvailable();
|
|
@@ -872,7 +872,7 @@ export function createEmbeddingService(config: EmbeddingConfig): IEmbeddingServi
|
|
|
872
872
|
return new TransformersEmbeddingService(config as TransformersEmbeddingConfig);
|
|
873
873
|
case 'mock':
|
|
874
874
|
return new MockEmbeddingService(config as MockEmbeddingConfig);
|
|
875
|
-
case '
|
|
875
|
+
case 'agentic-flow':
|
|
876
876
|
return new AgenticFlowEmbeddingService(config as AgenticFlowEmbeddingConfig);
|
|
877
877
|
default:
|
|
878
878
|
console.warn(`Unknown provider, using mock`);
|
|
@@ -884,13 +884,13 @@ export function createEmbeddingService(config: EmbeddingConfig): IEmbeddingServi
|
|
|
884
884
|
* Extended config with auto provider option
|
|
885
885
|
*/
|
|
886
886
|
export interface AutoEmbeddingConfig {
|
|
887
|
-
/** Provider: 'auto' will pick best available (
|
|
887
|
+
/** Provider: 'auto' will pick best available (agentic-flow > transformers > mock) */
|
|
888
888
|
provider: EmbeddingProvider | 'auto';
|
|
889
889
|
/** Fallback provider if primary fails */
|
|
890
890
|
fallback?: EmbeddingProvider;
|
|
891
|
-
/** Auto-install
|
|
891
|
+
/** Auto-install agentic-flow if not available (default: true for 'auto' provider) */
|
|
892
892
|
autoInstall?: boolean;
|
|
893
|
-
/** Model ID for
|
|
893
|
+
/** Model ID for agentic-flow */
|
|
894
894
|
modelId?: string;
|
|
895
895
|
/** Model name for transformers */
|
|
896
896
|
model?: string;
|
|
@@ -906,7 +906,7 @@ export interface AutoEmbeddingConfig {
|
|
|
906
906
|
* Create embedding service with automatic provider detection and fallback
|
|
907
907
|
*
|
|
908
908
|
* Features:
|
|
909
|
-
* - 'auto' provider picks best available:
|
|
909
|
+
* - 'auto' provider picks best available: agentic-flow > transformers > mock
|
|
910
910
|
* - Automatic fallback if primary provider fails to initialize
|
|
911
911
|
* - Pre-validates provider availability before returning
|
|
912
912
|
*
|
|
@@ -914,9 +914,9 @@ export interface AutoEmbeddingConfig {
|
|
|
914
914
|
* // Auto-select best provider
|
|
915
915
|
* const service = await createEmbeddingServiceAsync({ provider: 'auto' });
|
|
916
916
|
*
|
|
917
|
-
* // Try
|
|
917
|
+
* // Try agentic-flow, fallback to transformers
|
|
918
918
|
* const service = await createEmbeddingServiceAsync({
|
|
919
|
-
* provider: '
|
|
919
|
+
* provider: 'agentic-flow',
|
|
920
920
|
* fallback: 'transformers'
|
|
921
921
|
* });
|
|
922
922
|
*/
|
|
@@ -927,7 +927,7 @@ export async function createEmbeddingServiceAsync(
|
|
|
927
927
|
|
|
928
928
|
// Auto provider selection
|
|
929
929
|
if (provider === 'auto') {
|
|
930
|
-
// Try
|
|
930
|
+
// Try agentic-flow first (fastest, ONNX-based)
|
|
931
931
|
let agenticFlowAvailable = await isAgenticFlowAvailable();
|
|
932
932
|
|
|
933
933
|
// Auto-install if not available and autoInstall is enabled
|
|
@@ -938,7 +938,7 @@ export async function createEmbeddingServiceAsync(
|
|
|
938
938
|
if (agenticFlowAvailable) {
|
|
939
939
|
try {
|
|
940
940
|
const service = new AgenticFlowEmbeddingService({
|
|
941
|
-
provider: '
|
|
941
|
+
provider: 'agentic-flow',
|
|
942
942
|
modelId: rest.modelId ?? 'all-MiniLM-L6-v2',
|
|
943
943
|
dimensions: rest.dimensions ?? 384,
|
|
944
944
|
cacheSize: rest.cacheSize,
|
|
@@ -966,7 +966,7 @@ export async function createEmbeddingServiceAsync(
|
|
|
966
966
|
}
|
|
967
967
|
|
|
968
968
|
// Fallback to mock (always works)
|
|
969
|
-
console.warn('[embeddings] Using mock provider - install
|
|
969
|
+
console.warn('[embeddings] Using mock provider - install agentic-flow or @xenova/transformers for real embeddings');
|
|
970
970
|
return new MockEmbeddingService({
|
|
971
971
|
dimensions: rest.dimensions ?? 384,
|
|
972
972
|
cacheSize: rest.cacheSize,
|
|
@@ -976,9 +976,9 @@ export async function createEmbeddingServiceAsync(
|
|
|
976
976
|
// Specific provider with optional fallback
|
|
977
977
|
const createPrimary = (): IEmbeddingService => {
|
|
978
978
|
switch (provider) {
|
|
979
|
-
case '
|
|
979
|
+
case 'agentic-flow':
|
|
980
980
|
return new AgenticFlowEmbeddingService({
|
|
981
|
-
provider: '
|
|
981
|
+
provider: 'agentic-flow',
|
|
982
982
|
modelId: rest.modelId ?? 'all-MiniLM-L6-v2',
|
|
983
983
|
dimensions: rest.dimensions ?? 384,
|
|
984
984
|
cacheSize: rest.cacheSize,
|
package/src/index.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* V3 Embedding Service Module
|
|
3
3
|
*
|
|
4
|
-
* Production embedding service aligned with
|
|
4
|
+
* Production embedding service aligned with agentic-flow@alpha:
|
|
5
5
|
* - OpenAI provider (text-embedding-3-small/large)
|
|
6
6
|
* - Transformers.js provider (local ONNX models)
|
|
7
7
|
* - Agentic-flow provider (optimized ONNX with SIMD)
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Neural Substrate Integration
|
|
3
3
|
*
|
|
4
|
-
* Integrates
|
|
4
|
+
* Integrates agentic-flow's neural embedding features:
|
|
5
5
|
* - Semantic drift detection
|
|
6
6
|
* - Memory physics (hippocampal dynamics)
|
|
7
7
|
* - Embedding state machine
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* These features treat embeddings as a synthetic nervous system.
|
|
12
12
|
*/
|
|
13
13
|
|
|
14
|
-
// Types from
|
|
14
|
+
// Types from agentic-flow/embeddings
|
|
15
15
|
export interface DriftResult {
|
|
16
16
|
distance: number;
|
|
17
17
|
velocity: number;
|
|
@@ -66,7 +66,7 @@ export interface NeuralSubstrateConfig {
|
|
|
66
66
|
/**
|
|
67
67
|
* Lazy-loaded Neural Substrate wrapper
|
|
68
68
|
*
|
|
69
|
-
* Wraps
|
|
69
|
+
* Wraps agentic-flow's NeuralSubstrate with graceful fallback
|
|
70
70
|
*/
|
|
71
71
|
export class NeuralEmbeddingService {
|
|
72
72
|
private substrate: any = null;
|
|
@@ -82,7 +82,7 @@ export class NeuralEmbeddingService {
|
|
|
82
82
|
if (this.initialized) return this.available;
|
|
83
83
|
|
|
84
84
|
try {
|
|
85
|
-
const { getNeuralSubstrate } = await import('
|
|
85
|
+
const { getNeuralSubstrate } = await import('agentic-flow/embeddings');
|
|
86
86
|
this.substrate = await getNeuralSubstrate(this.config);
|
|
87
87
|
await this.substrate.init();
|
|
88
88
|
this.available = true;
|
|
@@ -253,7 +253,7 @@ export function createNeuralService(config: NeuralSubstrateConfig = {}): NeuralE
|
|
|
253
253
|
*/
|
|
254
254
|
export async function isNeuralAvailable(): Promise<boolean> {
|
|
255
255
|
try {
|
|
256
|
-
await import('
|
|
256
|
+
await import('agentic-flow/embeddings');
|
|
257
257
|
return true;
|
|
258
258
|
} catch {
|
|
259
259
|
return false;
|
|
@@ -271,10 +271,10 @@ export async function listEmbeddingModels(): Promise<Array<{
|
|
|
271
271
|
downloaded: boolean;
|
|
272
272
|
}>> {
|
|
273
273
|
try {
|
|
274
|
-
const { listAvailableModels } = await import('
|
|
274
|
+
const { listAvailableModels } = await import('agentic-flow/embeddings');
|
|
275
275
|
return listAvailableModels();
|
|
276
276
|
} catch {
|
|
277
|
-
// Return default models if
|
|
277
|
+
// Return default models if agentic-flow not available
|
|
278
278
|
return [
|
|
279
279
|
{ id: 'all-MiniLM-L6-v2', dimension: 384, size: '23MB', quantized: false, downloaded: false },
|
|
280
280
|
{ id: 'all-mpnet-base-v2', dimension: 768, size: '110MB', quantized: false, downloaded: false },
|
|
@@ -290,6 +290,6 @@ export async function downloadEmbeddingModel(
|
|
|
290
290
|
targetDir?: string,
|
|
291
291
|
onProgress?: (progress: { percent: number; bytesDownloaded: number; totalBytes: number }) => void
|
|
292
292
|
): Promise<string> {
|
|
293
|
-
const { downloadModel } = await import('
|
|
293
|
+
const { downloadModel } = await import('agentic-flow/embeddings');
|
|
294
294
|
return downloadModel(modelId, targetDir ?? '.models', onProgress);
|
|
295
295
|
}
|
package/src/types.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* V3 Embedding Service Types
|
|
3
3
|
*
|
|
4
|
-
* Type definitions for embedding service aligned with
|
|
4
|
+
* Type definitions for embedding service aligned with agentic-flow@alpha:
|
|
5
5
|
* - OpenAI provider
|
|
6
6
|
* - Transformers.js provider
|
|
7
7
|
* - Mock provider
|
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
/**
|
|
20
20
|
* Supported embedding providers
|
|
21
21
|
*/
|
|
22
|
-
export type EmbeddingProvider = 'openai' | 'transformers' | 'mock' | '
|
|
22
|
+
export type EmbeddingProvider = 'openai' | 'transformers' | 'mock' | 'agentic-flow';
|
|
23
23
|
|
|
24
24
|
/**
|
|
25
25
|
* Normalization type for embeddings
|
|
@@ -126,7 +126,7 @@ export interface MockEmbeddingConfig extends EmbeddingBaseConfig {
|
|
|
126
126
|
* - Pre-allocated buffers (no GC pressure)
|
|
127
127
|
*/
|
|
128
128
|
export interface AgenticFlowEmbeddingConfig extends EmbeddingBaseConfig {
|
|
129
|
-
provider: '
|
|
129
|
+
provider: 'agentic-flow';
|
|
130
130
|
|
|
131
131
|
/** Model ID (default: all-MiniLM-L6-v2) */
|
|
132
132
|
modelId?: string;
|