@byted-las/contextlake-openclaw 1.0.0 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +2 -1
- package/dist/index.js +5 -5
- package/dist/src/client/lancedb.js +13 -4
- package/dist/src/commands/cli.d.ts +5 -2
- package/dist/src/commands/cli.js +94 -10
- package/dist/src/commands/index.d.ts +2 -1
- package/dist/src/commands/index.js +31 -35
- package/dist/src/commands/slashcmd.d.ts +8 -1
- package/dist/src/commands/slashcmd.js +90 -6
- package/dist/src/commands/tools.d.ts +10 -218
- package/dist/src/commands/tools.js +109 -104
- package/dist/src/lib/actions/ingest-source.d.ts +15 -0
- package/dist/src/lib/actions/ingest-source.js +193 -0
- package/dist/src/lib/actions/ingest.d.ts +14 -7
- package/dist/src/lib/actions/ingest.js +133 -63
- package/dist/src/lib/actions/las-api.d.ts +13 -0
- package/dist/src/lib/actions/las-api.js +105 -0
- package/dist/src/lib/actions/las-tools.d.ts +3 -0
- package/dist/src/lib/actions/las-tools.js +194 -0
- package/dist/src/lib/actions/las.d.ts +64 -0
- package/dist/src/lib/actions/las.js +72 -0
- package/dist/src/lib/actions/manage.d.ts +3 -2
- package/dist/src/{skills/las-data-profiler/index.d.ts → lib/actions/profiler.d.ts} +4 -2
- package/dist/src/{skills/las-data-profiler/index.js → lib/actions/profiler.js} +19 -3
- package/dist/src/lib/actions/retrieve.d.ts +2 -1
- package/dist/src/lib/actions/retrieve.js +2 -18
- package/{src/skills/las-data-profiler → dist/src/lib/scripts}/s3_catalog.py +10 -1
- package/dist/src/processor/loader.js +9 -2
- package/dist/src/service/embedding/factory.js +1 -10
- package/dist/src/service/embedding/interface.d.ts +8 -1
- package/dist/src/service/embedding/local.js +16 -13
- package/dist/src/service/embedding/remote.d.ts +7 -0
- package/dist/src/service/embedding/remote.js +108 -7
- package/dist/src/service/metadata/interface.d.ts +1 -0
- package/dist/src/service/metadata/local.d.ts +1 -0
- package/dist/src/service/metadata/local.js +6 -0
- package/dist/src/skills/SKILL.md +174 -0
- package/dist/src/skills/contextlake-delete/SKILL.md +36 -0
- package/dist/src/skills/contextlake-ingest/SKILL.md +40 -0
- package/dist/src/skills/contextlake-list/SKILL.md +22 -0
- package/dist/src/skills/contextlake-retrieve/SKILL.md +37 -0
- package/dist/src/skills/las-data-profiler/SKILL.md +174 -0
- package/dist/src/utils/config.d.ts +34 -1
- package/dist/src/utils/config.js +16 -3
- package/dist/src/utils/credentials.d.ts +8 -0
- package/dist/src/utils/credentials.js +77 -0
- package/index.ts +8 -8
- package/openclaw.plugin.json +1 -1
- package/package.json +8 -7
- package/src/client/lancedb.ts +32 -21
- package/src/commands/cli.ts +105 -13
- package/src/commands/index.ts +45 -42
- package/src/commands/slashcmd.ts +69 -10
- package/src/commands/tools.ts +142 -117
- package/src/lib/actions/ingest.ts +151 -75
- package/src/lib/actions/las-api.ts +119 -0
- package/src/lib/actions/las-tools.ts +196 -0
- package/src/lib/actions/manage.ts +6 -5
- package/src/{skills/las-data-profiler/index.ts → lib/actions/profiler.ts} +21 -4
- package/src/lib/actions/retrieve.ts +16 -34
- package/src/lib/scripts/s3_catalog.py +617 -0
- package/src/processor/loader.ts +12 -4
- package/src/service/embedding/factory.ts +1 -8
- package/src/service/embedding/interface.ts +9 -1
- package/src/service/embedding/remote.ts +133 -13
- package/src/service/metadata/interface.ts +1 -0
- package/src/service/metadata/local.ts +7 -0
- package/src/service/storage/factory.ts +2 -2
- package/src/utils/config.ts +61 -8
- package/src/utils/credentials.ts +50 -0
- package/bin/contextlake-openclaw.js +0 -5
- package/dist/src/skills/las-data-profiler/register.d.ts +0 -1
- package/dist/src/skills/las-data-profiler/register.js +0 -19
- package/src/service/embedding/local.ts +0 -118
- package/src/skills/las-data-profiler/register.ts +0 -19
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
import { createStorageProvider } from '../../service/storage/factory';
|
|
2
2
|
import { createMetadataProvider } from '../../service/metadata/factory';
|
|
3
|
+
import { ContextLakeConfig } from '../../utils/config';
|
|
3
4
|
|
|
4
5
|
interface RetrieveParams {
|
|
5
6
|
query: string;
|
|
6
7
|
top_k?: number;
|
|
7
|
-
filter?: string;
|
|
8
|
+
filter?: string;
|
|
8
9
|
include_binary?: boolean;
|
|
9
10
|
}
|
|
10
11
|
|
|
11
|
-
export async function retrieveAssets(params: RetrieveParams, config:
|
|
12
|
+
export async function retrieveAssets(params: RetrieveParams, config: ContextLakeConfig, logger?: any) {
|
|
12
13
|
if (logger) {
|
|
13
14
|
logger.info(`[ContextLake-Action] Calling retrieveAssets with params: ${JSON.stringify(params)}`);
|
|
14
15
|
} else {
|
|
@@ -17,46 +18,37 @@ export async function retrieveAssets(params: RetrieveParams, config: any, logger
|
|
|
17
18
|
}
|
|
18
19
|
|
|
19
20
|
const metaConfig = config.metadata_storage || { type: 'local', lancedb_uri: './data/contextlake' };
|
|
20
|
-
const metadataProvider = createMetadataProvider(metaConfig);
|
|
21
|
+
const metadataProvider = createMetadataProvider(metaConfig as any);
|
|
21
22
|
|
|
22
23
|
let storageProvider: any;
|
|
23
24
|
if (config.file_storage && config.file_storage.type) {
|
|
24
|
-
storageProvider = createStorageProvider(config.file_storage);
|
|
25
|
+
storageProvider = createStorageProvider(config.file_storage as any);
|
|
25
26
|
}
|
|
26
27
|
|
|
27
28
|
await metadataProvider.connect();
|
|
28
29
|
|
|
29
30
|
const results = await metadataProvider.search(
|
|
30
|
-
params.query,
|
|
31
|
-
params.top_k || 5,
|
|
31
|
+
params.query,
|
|
32
|
+
params.top_k || 5,
|
|
32
33
|
params.filter
|
|
33
34
|
);
|
|
34
|
-
|
|
35
|
-
// Post-process to handle binary data retrieval if requested
|
|
36
35
|
|
|
37
36
|
if (params.include_binary) {
|
|
38
37
|
const enrichedResults = await Promise.all(results.map(async (doc: any) => {
|
|
39
|
-
|
|
40
|
-
const { binary_data, ...rest } = doc;
|
|
38
|
+
const { binary_data, vector, ...rest } = doc;
|
|
41
39
|
|
|
42
|
-
if (rest.vector) {
|
|
43
|
-
rest.vector = Array.from(rest.vector);
|
|
44
|
-
}
|
|
45
|
-
|
|
46
40
|
try {
|
|
47
41
|
if (rest.storage_type === 'inline') {
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
binary_content: binary_data ? binary_data.toString('base64') : null
|
|
42
|
+
return {
|
|
43
|
+
...rest,
|
|
44
|
+
binary_content: binary_data ? binary_data.toString('base64') : null
|
|
52
45
|
};
|
|
53
46
|
} else if (rest.storage_type === 'local' || rest.storage_type === 'tos') {
|
|
54
47
|
if (storageProvider) {
|
|
55
|
-
// Download from storage provider
|
|
56
48
|
const buffer = await storageProvider.downloadFile(rest.url);
|
|
57
|
-
return {
|
|
58
|
-
...rest,
|
|
59
|
-
binary_content: buffer.toString('base64')
|
|
49
|
+
return {
|
|
50
|
+
...rest,
|
|
51
|
+
binary_content: buffer.toString('base64')
|
|
60
52
|
};
|
|
61
53
|
} else {
|
|
62
54
|
return { ...rest, binary_error: 'Storage provider not configured' };
|
|
@@ -70,21 +62,11 @@ export async function retrieveAssets(params: RetrieveParams, config: any, logger
|
|
|
70
62
|
return enrichedResults;
|
|
71
63
|
}
|
|
72
64
|
|
|
73
|
-
// Remove binary_data from default response to keep it light and serializable
|
|
74
65
|
return results.map((doc: any) => {
|
|
75
|
-
const { binary_data, ...rest } = doc;
|
|
66
|
+
const { binary_data, vector, ...rest } = doc;
|
|
76
67
|
|
|
77
|
-
// Ensure all fields are standard serializable types (remove complex objects/buffers)
|
|
78
|
-
if (rest.vector) {
|
|
79
|
-
// Keep vector as standard array of numbers if it exists, but some clients might choke on large arrays.
|
|
80
|
-
// Usually vector is fine, but LanceDB might return custom objects like Float32Array which fail structuredClone in postMessage
|
|
81
|
-
rest.vector = Array.from(rest.vector);
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
// LanceDB records might be Proxy objects or contain non-clonable getters.
|
|
85
|
-
// We stringify and parse to get a clean, plain Javascript Object.
|
|
86
68
|
const safeDoc = JSON.parse(JSON.stringify(rest));
|
|
87
69
|
|
|
88
70
|
return safeDoc;
|
|
89
71
|
});
|
|
90
|
-
}
|
|
72
|
+
}
|