@byted-las/contextlake-openclaw 1.0.2 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/client/lancedb.js +1 -1
- package/dist/src/commands/cli.d.ts +2 -1
- package/dist/src/commands/cli.js +31 -8
- package/dist/src/commands/index.js +25 -6
- package/dist/src/commands/slashcmd.d.ts +6 -0
- package/dist/src/commands/slashcmd.js +90 -6
- package/dist/src/commands/tools.d.ts +2 -0
- package/dist/src/commands/tools.js +44 -37
- package/dist/src/lib/actions/ingest-source.d.ts +15 -0
- package/dist/src/lib/actions/ingest-source.js +193 -0
- package/dist/src/lib/actions/ingest.d.ts +13 -7
- package/dist/src/lib/actions/ingest.js +133 -58
- package/dist/src/lib/actions/las-api.d.ts +13 -0
- package/dist/src/lib/actions/las-api.js +105 -0
- package/dist/src/lib/actions/las-tools.d.ts +3 -0
- package/dist/src/lib/actions/las-tools.js +194 -0
- package/dist/src/lib/actions/las.d.ts +64 -0
- package/dist/src/lib/actions/las.js +72 -0
- package/dist/src/lib/actions/profiler.d.ts +3 -0
- package/dist/src/lib/actions/profiler.js +17 -1
- package/dist/src/lib/actions/retrieve.js +2 -8
- package/dist/src/lib/scripts/s3_catalog.py +10 -1
- package/dist/src/service/embedding/factory.js +1 -10
- package/dist/src/service/embedding/interface.d.ts +5 -0
- package/dist/src/service/embedding/remote.d.ts +1 -0
- package/dist/src/service/embedding/remote.js +31 -0
- package/dist/src/service/metadata/interface.d.ts +1 -0
- package/dist/src/service/metadata/local.d.ts +1 -0
- package/dist/src/service/metadata/local.js +6 -0
- package/dist/src/utils/config.js +11 -2
- package/dist/src/utils/credentials.d.ts +8 -0
- package/dist/src/utils/credentials.js +77 -0
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -5
- package/src/client/lancedb.ts +1 -1
- package/src/commands/cli.ts +35 -9
- package/src/commands/index.ts +30 -6
- package/src/commands/slashcmd.ts +67 -7
- package/src/commands/tools.ts +49 -41
- package/src/lib/actions/ingest.ts +151 -71
- package/src/lib/actions/las-api.ts +119 -0
- package/src/lib/actions/las-tools.ts +196 -0
- package/src/lib/actions/profiler.ts +18 -1
- package/src/lib/actions/retrieve.ts +2 -10
- package/src/lib/scripts/s3_catalog.py +10 -1
- package/src/service/embedding/factory.ts +1 -8
- package/src/service/embedding/interface.ts +6 -0
- package/src/service/embedding/remote.ts +36 -0
- package/src/service/metadata/interface.ts +1 -0
- package/src/service/metadata/local.ts +7 -0
- package/src/utils/config.ts +13 -2
- package/src/utils/credentials.ts +50 -0
- package/bin/contextlake-openclaw.js +0 -5
- package/src/service/embedding/local.ts +0 -121
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.lasPdfParseDoubao = lasPdfParseDoubao;
|
|
4
|
+
exports.lasLongVideoUnderstand = lasLongVideoUnderstand;
|
|
5
|
+
exports.lasBareImageTextEmbedding = lasBareImageTextEmbedding;
|
|
6
|
+
exports.lasSeed20 = lasSeed20;
|
|
7
|
+
exports.lasAsrPro = lasAsrPro;
|
|
8
|
+
exports.lasAudioExtractAndSplit = lasAudioExtractAndSplit;
|
|
9
|
+
function getLASConfig(config) {
|
|
10
|
+
// Attempt to get from env vars or config
|
|
11
|
+
const endpoint = process.env.LAS_ENDPOINT || (config?.las)?.endpoint;
|
|
12
|
+
const apiKey = process.env.LAS_API_KEY || (config?.las)?.api_key;
|
|
13
|
+
if (!endpoint || !apiKey) {
|
|
14
|
+
throw new Error("LAS_ENDPOINT and LAS_API_KEY must be set in environment variables or config");
|
|
15
|
+
}
|
|
16
|
+
return { endpoint, apiKey };
|
|
17
|
+
}
|
|
18
|
+
async function lasFetch(path, payload, config) {
|
|
19
|
+
const { endpoint, apiKey } = getLASConfig(config);
|
|
20
|
+
const url = `${endpoint.replace(/\/$/, '')}${path}`;
|
|
21
|
+
const response = await fetch(url, {
|
|
22
|
+
method: 'POST',
|
|
23
|
+
headers: {
|
|
24
|
+
'Content-Type': 'application/json',
|
|
25
|
+
'Authorization': `Bearer ${apiKey}`
|
|
26
|
+
},
|
|
27
|
+
body: JSON.stringify(payload)
|
|
28
|
+
});
|
|
29
|
+
if (!response.ok) {
|
|
30
|
+
let errorText = await response.text().catch(() => '');
|
|
31
|
+
throw new Error(`LAS API Error: ${response.status} ${response.statusText} - ${errorText}`);
|
|
32
|
+
}
|
|
33
|
+
return await response.json();
|
|
34
|
+
}
|
|
35
|
+
async function lasPdfParseDoubao(params, config) {
|
|
36
|
+
return lasFetch('/api/v1/submit', {
|
|
37
|
+
operator_id: 'las_pdf_parse_doubao',
|
|
38
|
+
operator_version: 'v1',
|
|
39
|
+
data: params
|
|
40
|
+
}, config);
|
|
41
|
+
}
|
|
42
|
+
async function lasLongVideoUnderstand(params, config) {
|
|
43
|
+
return lasFetch('/api/v1/submit', {
|
|
44
|
+
operator_id: 'las_long_video_understand',
|
|
45
|
+
operator_version: 'v1',
|
|
46
|
+
data: params
|
|
47
|
+
}, config);
|
|
48
|
+
}
|
|
49
|
+
async function lasBareImageTextEmbedding(params, config) {
|
|
50
|
+
return lasFetch('/api/v1/embeddings/multimodal', {
|
|
51
|
+
model: 'doubao-embedding-vision',
|
|
52
|
+
input: params.input,
|
|
53
|
+
encoding_format: params.encoding_format
|
|
54
|
+
}, config);
|
|
55
|
+
}
|
|
56
|
+
async function lasSeed20(params, config) {
|
|
57
|
+
return lasFetch('/api/v1/chat/completions', params, config);
|
|
58
|
+
}
|
|
59
|
+
async function lasAsrPro(params, config) {
|
|
60
|
+
return lasFetch('/api/v1/submit', {
|
|
61
|
+
operator_id: 'las_asr_pro',
|
|
62
|
+
operator_version: 'v1',
|
|
63
|
+
data: params
|
|
64
|
+
}, config);
|
|
65
|
+
}
|
|
66
|
+
async function lasAudioExtractAndSplit(params, config) {
|
|
67
|
+
return lasFetch('/api/v1/process', {
|
|
68
|
+
operator_id: 'las_audio_extract_and_split',
|
|
69
|
+
operator_version: 'v1',
|
|
70
|
+
data: params
|
|
71
|
+
}, config);
|
|
72
|
+
}
|
|
@@ -34,6 +34,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
36
|
exports.connectDataSource = connectDataSource;
|
|
37
|
+
exports.listDataSources = listDataSources;
|
|
37
38
|
const path = __importStar(require("path"));
|
|
38
39
|
const fs = __importStar(require("fs"));
|
|
39
40
|
const os = __importStar(require("os"));
|
|
@@ -41,7 +42,7 @@ const child_process_1 = require("child_process");
|
|
|
41
42
|
// ---------------------------------------------------------------------------
|
|
42
43
|
// Constants
|
|
43
44
|
// ---------------------------------------------------------------------------
|
|
44
|
-
const BASE_DIR = path.join(os.homedir(), '.openclaw', '
|
|
45
|
+
const BASE_DIR = path.join(os.homedir(), '.openclaw', 'contextlake', 'profiler');
|
|
45
46
|
const PYTHON_DEPS = ['boto3', 'lancedb', 'pyarrow', 'pandas', 'Pillow', 'mutagen', 'pymupdf'];
|
|
46
47
|
// ---------------------------------------------------------------------------
|
|
47
48
|
// Helpers
|
|
@@ -229,3 +230,18 @@ async function connectDataSource(params, _ctx) {
|
|
|
229
230
|
});
|
|
230
231
|
});
|
|
231
232
|
}
|
|
233
|
+
async function listDataSources(_ctx) {
|
|
234
|
+
try {
|
|
235
|
+
if (!fs.existsSync(BASE_DIR)) {
|
|
236
|
+
return { datasources: [] };
|
|
237
|
+
}
|
|
238
|
+
const entries = fs.readdirSync(BASE_DIR, { withFileTypes: true });
|
|
239
|
+
const datasources = entries
|
|
240
|
+
.filter(entry => entry.isDirectory())
|
|
241
|
+
.map(entry => entry.name);
|
|
242
|
+
return { datasources };
|
|
243
|
+
}
|
|
244
|
+
catch (error) {
|
|
245
|
+
throw new Error(`Failed to list data sources: ${error.message}`);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
@@ -21,10 +21,7 @@ async function retrieveAssets(params, config, logger) {
|
|
|
21
21
|
const results = await metadataProvider.search(params.query, params.top_k || 5, params.filter);
|
|
22
22
|
if (params.include_binary) {
|
|
23
23
|
const enrichedResults = await Promise.all(results.map(async (doc) => {
|
|
24
|
-
const { binary_data, ...rest } = doc;
|
|
25
|
-
if (rest.vector) {
|
|
26
|
-
rest.vector = Array.from(rest.vector);
|
|
27
|
-
}
|
|
24
|
+
const { binary_data, vector, ...rest } = doc;
|
|
28
25
|
try {
|
|
29
26
|
if (rest.storage_type === 'inline') {
|
|
30
27
|
return {
|
|
@@ -53,10 +50,7 @@ async function retrieveAssets(params, config, logger) {
|
|
|
53
50
|
return enrichedResults;
|
|
54
51
|
}
|
|
55
52
|
return results.map((doc) => {
|
|
56
|
-
const { binary_data, ...rest } = doc;
|
|
57
|
-
if (rest.vector) {
|
|
58
|
-
rest.vector = Array.from(rest.vector);
|
|
59
|
-
}
|
|
53
|
+
const { binary_data, vector, ...rest } = doc;
|
|
60
54
|
const safeDoc = JSON.parse(JSON.stringify(rest));
|
|
61
55
|
return safeDoc;
|
|
62
56
|
});
|
|
@@ -523,10 +523,19 @@ def main():
|
|
|
523
523
|
parser.add_argument('--region', default='')
|
|
524
524
|
parser.add_argument('--bucket', required=True)
|
|
525
525
|
parser.add_argument('--prefix', required=True)
|
|
526
|
-
parser.add_argument('--db-path', default='
|
|
526
|
+
parser.add_argument('--db-path', default=None, help='Path to LanceDB database. Defaults to ~/.openclaw/las-data-profiler/{datasource_name}/catalog_db if datasource_name is provided.')
|
|
527
|
+
parser.add_argument('--datasource-name', default='', help='Name of the datasource. Used to determine default db-path if not explicitly provided.')
|
|
527
528
|
parser.add_argument('--sample-rows', type=int, default=100)
|
|
528
529
|
args = parser.parse_args()
|
|
529
530
|
|
|
531
|
+
if not args.db_path:
|
|
532
|
+
if args.datasource_name:
|
|
533
|
+
import os
|
|
534
|
+
home_dir = os.path.expanduser('~')
|
|
535
|
+
args.db_path = os.path.join(home_dir, '.openclaw', 'las-data-profiler', args.datasource_name, 'catalog_db')
|
|
536
|
+
else:
|
|
537
|
+
args.db_path = './catalog_db'
|
|
538
|
+
|
|
530
539
|
print(f"[las-data-profiler] vendor={args.vendor}, bucket={args.bucket}, prefix={args.prefix}")
|
|
531
540
|
print(f"[las-data-profiler] db_path={args.db_path}")
|
|
532
541
|
|
|
@@ -1,16 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.createEmbeddingProvider = createEmbeddingProvider;
|
|
4
|
-
const local_1 = require("./local");
|
|
5
4
|
const remote_1 = require("./remote");
|
|
6
5
|
function createEmbeddingProvider(config) {
|
|
7
|
-
|
|
8
|
-
return new local_1.LocalEmbeddingProvider(config);
|
|
9
|
-
}
|
|
10
|
-
else if (config.provider === 'openai' || config.provider === 'remote' || config.provider === 'las') {
|
|
11
|
-
return new remote_1.RemoteEmbeddingProvider(config);
|
|
12
|
-
}
|
|
13
|
-
else {
|
|
14
|
-
throw new Error(`Unsupported embedding provider: ${config.provider}`);
|
|
15
|
-
}
|
|
6
|
+
return new remote_1.RemoteEmbeddingProvider(config);
|
|
16
7
|
}
|
|
@@ -9,6 +9,11 @@ export interface EmbeddingProvider {
|
|
|
9
9
|
* @param texts - Array of input texts
|
|
10
10
|
*/
|
|
11
11
|
generateEmbeddings(texts: string[]): Promise<number[][]>;
|
|
12
|
+
/**
|
|
13
|
+
* Generate embedding for multimodal input (LAS specific)
|
|
14
|
+
* @param input - Multimodal input array
|
|
15
|
+
*/
|
|
16
|
+
generateMultimodalEmbedding?(input: any[]): Promise<number[]>;
|
|
12
17
|
}
|
|
13
18
|
export interface EmbeddingConfig {
|
|
14
19
|
provider: 'local' | 'remote' | 'openai' | 'las';
|
|
@@ -12,4 +12,5 @@ export declare class RemoteEmbeddingProvider implements EmbeddingProvider {
|
|
|
12
12
|
generateEmbeddings(texts: string[]): Promise<number[][]>;
|
|
13
13
|
private generateOpenAICompatibleEmbeddings;
|
|
14
14
|
private generateLasEmbeddings;
|
|
15
|
+
generateMultimodalEmbedding(input: any[]): Promise<number[]>;
|
|
15
16
|
}
|
|
@@ -108,5 +108,36 @@ class RemoteEmbeddingProvider {
|
|
|
108
108
|
});
|
|
109
109
|
return Promise.all(requests);
|
|
110
110
|
}
|
|
111
|
+
async generateMultimodalEmbedding(input) {
|
|
112
|
+
if (this.mode !== 'las-multimodal') {
|
|
113
|
+
throw new Error('generateMultimodalEmbedding requires LAS multimodal provider');
|
|
114
|
+
}
|
|
115
|
+
const endpoint = this.apiBase.endsWith('/api/v1/embeddings/multimodal')
|
|
116
|
+
? this.apiBase
|
|
117
|
+
: `${this.apiBase}/api/v1/embeddings/multimodal`;
|
|
118
|
+
const response = await fetch(endpoint, {
|
|
119
|
+
method: 'POST',
|
|
120
|
+
headers: {
|
|
121
|
+
'Content-Type': 'application/json',
|
|
122
|
+
'Authorization': `Bearer ${this.apiKey}`
|
|
123
|
+
},
|
|
124
|
+
body: JSON.stringify({
|
|
125
|
+
model: this.modelName,
|
|
126
|
+
encoding_format: this.encodingFormat,
|
|
127
|
+
...(this.dimensions ? { dimensions: this.dimensions } : {}),
|
|
128
|
+
input
|
|
129
|
+
})
|
|
130
|
+
});
|
|
131
|
+
if (!response.ok) {
|
|
132
|
+
const error = await response.text();
|
|
133
|
+
throw new Error(`LAS embedding API error: ${response.status} ${error}`);
|
|
134
|
+
}
|
|
135
|
+
const data = await response.json();
|
|
136
|
+
const embedding = data?.data?.embedding;
|
|
137
|
+
if (!Array.isArray(embedding) || embedding.length === 0) {
|
|
138
|
+
throw new Error('Unexpected LAS embedding response format');
|
|
139
|
+
}
|
|
140
|
+
return embedding;
|
|
141
|
+
}
|
|
111
142
|
}
|
|
112
143
|
exports.RemoteEmbeddingProvider = RemoteEmbeddingProvider;
|
|
@@ -7,6 +7,7 @@ export interface MetadataProvider {
|
|
|
7
7
|
list(limit?: number, filter?: string): Promise<DocumentSchema[]>;
|
|
8
8
|
delete(filter: string): Promise<void>;
|
|
9
9
|
generateEmbedding(text: string): Promise<number[]>;
|
|
10
|
+
generateMultimodalEmbedding?(input: any[]): Promise<number[]>;
|
|
10
11
|
}
|
|
11
12
|
export interface MetadataConfig {
|
|
12
13
|
type: 'local' | 'remote';
|
|
@@ -10,4 +10,5 @@ export declare class LocalMetadataProvider implements MetadataProvider {
|
|
|
10
10
|
list(limit?: number, filter?: string): Promise<DocumentSchema[]>;
|
|
11
11
|
delete(filter: string): Promise<void>;
|
|
12
12
|
generateEmbedding(text: string): Promise<number[]>;
|
|
13
|
+
generateMultimodalEmbedding(input: any[]): Promise<number[]>;
|
|
13
14
|
}
|
|
@@ -45,5 +45,11 @@ class LocalMetadataProvider {
|
|
|
45
45
|
async generateEmbedding(text) {
|
|
46
46
|
return await this.embeddingProvider.generateEmbedding(text);
|
|
47
47
|
}
|
|
48
|
+
async generateMultimodalEmbedding(input) {
|
|
49
|
+
if (this.embeddingProvider.generateMultimodalEmbedding) {
|
|
50
|
+
return await this.embeddingProvider.generateMultimodalEmbedding(input);
|
|
51
|
+
}
|
|
52
|
+
throw new Error('generateMultimodalEmbedding not supported by this embedding provider');
|
|
53
|
+
}
|
|
48
54
|
}
|
|
49
55
|
exports.LocalMetadataProvider = LocalMetadataProvider;
|
package/dist/src/utils/config.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.getPluginConfig = getPluginConfig;
|
|
4
|
+
const credentials_1 = require("./credentials");
|
|
4
5
|
function getPluginConfig(ctx) {
|
|
6
|
+
const creds = (0, credentials_1.loadCredentials)();
|
|
5
7
|
return ctx.config?.plugins?.entries?.['contextlake-openclaw']?.config || {
|
|
6
8
|
metadata_storage: {
|
|
7
9
|
type: 'local',
|
|
@@ -9,12 +11,19 @@ function getPluginConfig(ctx) {
|
|
|
9
11
|
embedding: {
|
|
10
12
|
provider: 'las',
|
|
11
13
|
model_name: 'doubao-embedding-vision-250615',
|
|
12
|
-
api_key: process.env.LAS_API_KEY,
|
|
14
|
+
api_key: process.env.LAS_API_KEY || creds.LAS_API_KEY,
|
|
13
15
|
api_base: process.env.LAS_BASE_URL || 'https://operator.las.cn-beijing.volces.com',
|
|
14
16
|
dimensions: 2048,
|
|
15
17
|
encoding_format: 'float'
|
|
16
18
|
}
|
|
17
19
|
},
|
|
18
|
-
file_storage: {
|
|
20
|
+
file_storage: {
|
|
21
|
+
type: 'local',
|
|
22
|
+
local_base_dir: require('path').join(require('os').homedir(), '.openclaw', 'contextlake', 'files'),
|
|
23
|
+
tos: {
|
|
24
|
+
access_key: process.env.VOLCENGINE_ACCESS_KEY || creds.VOLCENGINE_ACCESS_KEY,
|
|
25
|
+
secret_key: process.env.VOLCENGINE_SECRET_KEY || creds.VOLCENGINE_SECRET_KEY
|
|
26
|
+
}
|
|
27
|
+
}
|
|
19
28
|
};
|
|
20
29
|
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export interface Credentials {
|
|
2
|
+
LAS_API_KEY?: string;
|
|
3
|
+
VOLCENGINE_ACCESS_KEY?: string;
|
|
4
|
+
VOLCENGINE_SECRET_KEY?: string;
|
|
5
|
+
}
|
|
6
|
+
export declare function loadCredentials(): Credentials;
|
|
7
|
+
export declare function saveCredentials(creds: Credentials): void;
|
|
8
|
+
export declare function promptForInput(promptText: string, defaultVal?: string): Promise<string>;
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.loadCredentials = loadCredentials;
|
|
37
|
+
exports.saveCredentials = saveCredentials;
|
|
38
|
+
exports.promptForInput = promptForInput;
|
|
39
|
+
const fs = __importStar(require("fs"));
|
|
40
|
+
const path = __importStar(require("path"));
|
|
41
|
+
const os = __importStar(require("os"));
|
|
42
|
+
const readline = __importStar(require("readline"));
|
|
43
|
+
const CONFIG_DIR = path.join(os.homedir(), '.openclaw', 'contextlake');
|
|
44
|
+
const CONFIG_FILE = path.join(CONFIG_DIR, 'credentials.json');
|
|
45
|
+
function loadCredentials() {
|
|
46
|
+
if (fs.existsSync(CONFIG_FILE)) {
|
|
47
|
+
try {
|
|
48
|
+
const content = fs.readFileSync(CONFIG_FILE, 'utf-8');
|
|
49
|
+
return JSON.parse(content);
|
|
50
|
+
}
|
|
51
|
+
catch (e) {
|
|
52
|
+
// ignore
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return {};
|
|
56
|
+
}
|
|
57
|
+
function saveCredentials(creds) {
|
|
58
|
+
if (!fs.existsSync(CONFIG_DIR)) {
|
|
59
|
+
fs.mkdirSync(CONFIG_DIR, { recursive: true });
|
|
60
|
+
}
|
|
61
|
+
const current = loadCredentials();
|
|
62
|
+
const updated = { ...current, ...creds };
|
|
63
|
+
fs.writeFileSync(CONFIG_FILE, JSON.stringify(updated, null, 2), { mode: 0o600 });
|
|
64
|
+
}
|
|
65
|
+
async function promptForInput(promptText, defaultVal) {
|
|
66
|
+
const rl = readline.createInterface({
|
|
67
|
+
input: process.stdin,
|
|
68
|
+
output: process.stdout,
|
|
69
|
+
});
|
|
70
|
+
const displayPrompt = defaultVal ? `${promptText} [${defaultVal}]: ` : `${promptText}: `;
|
|
71
|
+
return new Promise((resolve) => {
|
|
72
|
+
rl.question(displayPrompt, (answer) => {
|
|
73
|
+
rl.close();
|
|
74
|
+
resolve(answer.trim() || defaultVal || '');
|
|
75
|
+
});
|
|
76
|
+
});
|
|
77
|
+
}
|
package/openclaw.plugin.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"id": "contextlake-openclaw",
|
|
3
3
|
"name": "ContextLake",
|
|
4
|
-
"version": "1.
|
|
4
|
+
"version": "1.0.3",
|
|
5
5
|
"description": "A lightweight knowledge base plugin for OpenClaw using LanceDB and TOS, with data profiling support",
|
|
6
6
|
"skills": ["./src/skills"],
|
|
7
7
|
"configSchema": {
|
package/package.json
CHANGED
|
@@ -1,18 +1,14 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@byted-las/contextlake-openclaw",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.3",
|
|
4
4
|
"description": "ContextLake OpenClaw Plugin for managing knowledge base",
|
|
5
5
|
"main": "index.ts",
|
|
6
6
|
"files": [
|
|
7
7
|
"dist",
|
|
8
|
-
"bin",
|
|
9
8
|
"index.ts",
|
|
10
9
|
"src",
|
|
11
10
|
"openclaw.plugin.json"
|
|
12
11
|
],
|
|
13
|
-
"bin": {
|
|
14
|
-
"contextlake-openclaw": "./bin/contextlake-openclaw.js"
|
|
15
|
-
},
|
|
16
12
|
"openclaw": {
|
|
17
13
|
"extensions": [
|
|
18
14
|
"./dist/index.js"
|
package/src/client/lancedb.ts
CHANGED
|
@@ -87,7 +87,7 @@ export class ContextLakeLanceDBClient {
|
|
|
87
87
|
return await fallbackQuery.toArray();
|
|
88
88
|
}
|
|
89
89
|
|
|
90
|
-
const vector = await this.embeddingProvider.
|
|
90
|
+
const vector = await this.embeddingProvider.generateMultimodalEmbedding!([{ type: 'text', text: query }]);
|
|
91
91
|
// @ts-ignore
|
|
92
92
|
let search = table.vectorSearch(vector).limit(normalizedLimit);
|
|
93
93
|
if (filter) {
|
package/src/commands/cli.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
// @ts-ignore
|
|
2
|
-
import {
|
|
2
|
+
import { ingestSource } from '../lib/actions/ingest';
|
|
3
3
|
import { retrieveAssets } from '../lib/actions/retrieve';
|
|
4
4
|
import { listAssets, deleteAssets } from '../lib/actions/manage';
|
|
5
5
|
import { connectDataSource, ConnectParams } from '../lib/actions/profiler';
|
|
6
|
+
import { loadCredentials, saveCredentials, promptForInput } from '../utils/credentials';
|
|
6
7
|
import { ContextLakeConfig } from '../utils/config';
|
|
7
8
|
|
|
8
9
|
function parseOptionalInt(value: any, fallback: number): number {
|
|
@@ -72,15 +73,11 @@ export function getCliCommands(pluginConfig: ContextLakeConfig, logger: any) {
|
|
|
72
73
|
}
|
|
73
74
|
},
|
|
74
75
|
|
|
75
|
-
ingestAction: async (
|
|
76
|
-
logger.info(`[${new Date().toISOString()}] [ContextLake] CLI ingest started`, {
|
|
76
|
+
ingestAction: async (datasource_name: string) => {
|
|
77
|
+
logger.info(`[${new Date().toISOString()}] [ContextLake] CLI ingest started`, { datasource_name });
|
|
77
78
|
try {
|
|
78
|
-
const
|
|
79
|
-
|
|
80
|
-
files,
|
|
81
|
-
metadata,
|
|
82
|
-
chunkSize: parseOptionalInt(options.chunkSize, 500),
|
|
83
|
-
overlap: parseOptionalInt(options.overlap, 50)
|
|
79
|
+
const result = await ingestSource({
|
|
80
|
+
datasource_name
|
|
84
81
|
}, pluginConfig, logger);
|
|
85
82
|
// eslint-disable-next-line no-console
|
|
86
83
|
console.log(JSON.stringify(result, null, 2));
|
|
@@ -138,6 +135,35 @@ export function getCliCommands(pluginConfig: ContextLakeConfig, logger: any) {
|
|
|
138
135
|
console.error('Error:', e.message);
|
|
139
136
|
logger.error(`[${new Date().toISOString()}] [ContextLake] CLI delete failed`, { error: e.message, stack: e.stack });
|
|
140
137
|
}
|
|
138
|
+
},
|
|
139
|
+
|
|
140
|
+
onboardAction: async () => {
|
|
141
|
+
logger.info(`[${new Date().toISOString()}] [ContextLake] CLI onboard started`);
|
|
142
|
+
try {
|
|
143
|
+
const currentCreds = loadCredentials();
|
|
144
|
+
// eslint-disable-next-line no-console
|
|
145
|
+
console.log('Welcome to ContextLake Onboarding!');
|
|
146
|
+
// eslint-disable-next-line no-console
|
|
147
|
+
console.log('Please provide your credentials below. Press enter to keep the current value.');
|
|
148
|
+
|
|
149
|
+
const lasApiKey = await promptForInput('LAS_API_KEY', currentCreds.LAS_API_KEY);
|
|
150
|
+
const volcengineAccessKey = await promptForInput('VOLCENGINE_ACCESS_KEY', currentCreds.VOLCENGINE_ACCESS_KEY);
|
|
151
|
+
const volcengineSecretKey = await promptForInput('VOLCENGINE_SECRET_KEY', currentCreds.VOLCENGINE_SECRET_KEY);
|
|
152
|
+
|
|
153
|
+
const newCreds = {
|
|
154
|
+
LAS_API_KEY: lasApiKey,
|
|
155
|
+
VOLCENGINE_ACCESS_KEY: volcengineAccessKey,
|
|
156
|
+
VOLCENGINE_SECRET_KEY: volcengineSecretKey
|
|
157
|
+
};
|
|
158
|
+
|
|
159
|
+
saveCredentials(newCreds);
|
|
160
|
+
// eslint-disable-next-line no-console
|
|
161
|
+
console.log('Credentials saved successfully!');
|
|
162
|
+
logger.info(`[${new Date().toISOString()}] [ContextLake] CLI onboard success`);
|
|
163
|
+
} catch (e: any) {
|
|
164
|
+
console.error('Error during onboarding:', e.message);
|
|
165
|
+
logger.error(`[${new Date().toISOString()}] [ContextLake] CLI onboard failed`, { error: e.message, stack: e.stack });
|
|
166
|
+
}
|
|
141
167
|
}
|
|
142
168
|
};
|
|
143
169
|
}
|
package/src/commands/index.ts
CHANGED
|
@@ -27,6 +27,14 @@ export function registerAll(ctx: OpenClawPluginApi, logger: PluginLogger) {
|
|
|
27
27
|
ctx.registerTool(tools.lasDataProfilerTool );
|
|
28
28
|
logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${tools.lasDataProfilerTool.name}`);
|
|
29
29
|
|
|
30
|
+
ctx.registerTool(tools.listDatasourceTool );
|
|
31
|
+
logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${tools.listDatasourceTool.name}`);
|
|
32
|
+
|
|
33
|
+
for (const lasTool of tools.lasTools) {
|
|
34
|
+
ctx.registerTool(lasTool);
|
|
35
|
+
logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${lasTool.name}`);
|
|
36
|
+
}
|
|
37
|
+
|
|
30
38
|
} catch (error: any) {
|
|
31
39
|
logger.error(`[${new Date().toISOString()}] [ContextLake] Error registering agent tools: ${error.message}${error.stack ? '\\n' + error.stack : ''}`);
|
|
32
40
|
throw error;
|
|
@@ -56,11 +64,8 @@ export function registerAll(ctx: OpenClawPluginApi, logger: PluginLogger) {
|
|
|
56
64
|
.action(commands.connectAction);
|
|
57
65
|
|
|
58
66
|
// Ingest
|
|
59
|
-
contextlake.command('ingest <
|
|
60
|
-
.description('
|
|
61
|
-
.option('-c, --chunk-size <number>', 'Chunk size for text splitting', '500')
|
|
62
|
-
.option('-o, --overlap <number>', 'Chunk overlap size', '50')
|
|
63
|
-
.option('-m, --metadata <json>', 'JSON metadata to attach to the documents')
|
|
67
|
+
contextlake.command('ingest <datasource_name>')
|
|
68
|
+
.description('Process and ingest all files from a connected data source into the knowledge base')
|
|
64
69
|
.action(commands.ingestAction);
|
|
65
70
|
|
|
66
71
|
// Search
|
|
@@ -83,6 +88,11 @@ export function registerAll(ctx: OpenClawPluginApi, logger: PluginLogger) {
|
|
|
83
88
|
.option('--ids <ids...>', 'List of specific file IDs to delete')
|
|
84
89
|
.option('-f, --filter <string>', 'Filter string to match documents for deletion')
|
|
85
90
|
.action(commands.deleteAction);
|
|
91
|
+
|
|
92
|
+
// Onboard
|
|
93
|
+
contextlake.command('onboard')
|
|
94
|
+
.description('Configure credentials for ContextLake')
|
|
95
|
+
.action(commands.onboardAction);
|
|
86
96
|
|
|
87
97
|
}, { commands: ['contextlake'] });
|
|
88
98
|
logger.info(`[${new Date().toISOString()}] [ContextLake] CLI commands registered`);
|
|
@@ -102,7 +112,7 @@ export function registerAll(ctx: OpenClawPluginApi, logger: PluginLogger) {
|
|
|
102
112
|
|
|
103
113
|
ctx.registerCommand({
|
|
104
114
|
name: 'contextlake-ingest',
|
|
105
|
-
description: '
|
|
115
|
+
description: 'Process and ingest all files from a connected data source (usage: /contextlake-ingest <datasource_name>)',
|
|
106
116
|
acceptsArgs: true,
|
|
107
117
|
handler: slashCommands.ingestHandler
|
|
108
118
|
});
|
|
@@ -127,6 +137,20 @@ export function registerAll(ctx: OpenClawPluginApi, logger: PluginLogger) {
|
|
|
127
137
|
acceptsArgs: true,
|
|
128
138
|
handler: slashCommands.deleteHandler
|
|
129
139
|
});
|
|
140
|
+
|
|
141
|
+
ctx.registerCommand({
|
|
142
|
+
name: 'contextlake-profiler',
|
|
143
|
+
description: 'Connect to a data source and profile its structure (usage: /contextlake-profiler <datasource_name> <vendor> <bucket> <prefix>)',
|
|
144
|
+
acceptsArgs: true,
|
|
145
|
+
handler: slashCommands.profilerHandler
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
ctx.registerCommand({
|
|
149
|
+
name: 'contextlake-list-datasource',
|
|
150
|
+
description: 'List all connected and profiled data sources (usage: /contextlake-list-datasource)',
|
|
151
|
+
acceptsArgs: false,
|
|
152
|
+
handler: slashCommands.listDatasourceHandler
|
|
153
|
+
});
|
|
130
154
|
|
|
131
155
|
logger.info(`[${new Date().toISOString()}] [ContextLake] Slash commands registered`);
|
|
132
156
|
} catch (error: any) {
|