@byted-las/contextlake-openclaw 1.0.0 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +2 -1
- package/dist/index.js +5 -5
- package/dist/src/client/lancedb.js +13 -4
- package/dist/src/commands/cli.d.ts +5 -2
- package/dist/src/commands/cli.js +94 -10
- package/dist/src/commands/index.d.ts +2 -1
- package/dist/src/commands/index.js +31 -35
- package/dist/src/commands/slashcmd.d.ts +8 -1
- package/dist/src/commands/slashcmd.js +90 -6
- package/dist/src/commands/tools.d.ts +10 -218
- package/dist/src/commands/tools.js +109 -104
- package/dist/src/lib/actions/ingest-source.d.ts +15 -0
- package/dist/src/lib/actions/ingest-source.js +193 -0
- package/dist/src/lib/actions/ingest.d.ts +14 -7
- package/dist/src/lib/actions/ingest.js +133 -63
- package/dist/src/lib/actions/las-api.d.ts +13 -0
- package/dist/src/lib/actions/las-api.js +105 -0
- package/dist/src/lib/actions/las-tools.d.ts +3 -0
- package/dist/src/lib/actions/las-tools.js +194 -0
- package/dist/src/lib/actions/las.d.ts +64 -0
- package/dist/src/lib/actions/las.js +72 -0
- package/dist/src/lib/actions/manage.d.ts +3 -2
- package/dist/src/{skills/las-data-profiler/index.d.ts → lib/actions/profiler.d.ts} +4 -2
- package/dist/src/{skills/las-data-profiler/index.js → lib/actions/profiler.js} +19 -3
- package/dist/src/lib/actions/retrieve.d.ts +2 -1
- package/dist/src/lib/actions/retrieve.js +2 -18
- package/{src/skills/las-data-profiler → dist/src/lib/scripts}/s3_catalog.py +10 -1
- package/dist/src/processor/loader.js +9 -2
- package/dist/src/service/embedding/factory.js +1 -10
- package/dist/src/service/embedding/interface.d.ts +8 -1
- package/dist/src/service/embedding/local.js +16 -13
- package/dist/src/service/embedding/remote.d.ts +7 -0
- package/dist/src/service/embedding/remote.js +108 -7
- package/dist/src/service/metadata/interface.d.ts +1 -0
- package/dist/src/service/metadata/local.d.ts +1 -0
- package/dist/src/service/metadata/local.js +6 -0
- package/dist/src/skills/SKILL.md +174 -0
- package/dist/src/skills/contextlake-delete/SKILL.md +36 -0
- package/dist/src/skills/contextlake-ingest/SKILL.md +40 -0
- package/dist/src/skills/contextlake-list/SKILL.md +22 -0
- package/dist/src/skills/contextlake-retrieve/SKILL.md +37 -0
- package/dist/src/skills/las-data-profiler/SKILL.md +174 -0
- package/dist/src/utils/config.d.ts +34 -1
- package/dist/src/utils/config.js +16 -3
- package/dist/src/utils/credentials.d.ts +8 -0
- package/dist/src/utils/credentials.js +77 -0
- package/index.ts +8 -8
- package/openclaw.plugin.json +1 -1
- package/package.json +8 -7
- package/src/client/lancedb.ts +32 -21
- package/src/commands/cli.ts +105 -13
- package/src/commands/index.ts +45 -42
- package/src/commands/slashcmd.ts +69 -10
- package/src/commands/tools.ts +142 -117
- package/src/lib/actions/ingest.ts +151 -75
- package/src/lib/actions/las-api.ts +119 -0
- package/src/lib/actions/las-tools.ts +196 -0
- package/src/lib/actions/manage.ts +6 -5
- package/src/{skills/las-data-profiler/index.ts → lib/actions/profiler.ts} +21 -4
- package/src/lib/actions/retrieve.ts +16 -34
- package/src/lib/scripts/s3_catalog.py +617 -0
- package/src/processor/loader.ts +12 -4
- package/src/service/embedding/factory.ts +1 -8
- package/src/service/embedding/interface.ts +9 -1
- package/src/service/embedding/remote.ts +133 -13
- package/src/service/metadata/interface.ts +1 -0
- package/src/service/metadata/local.ts +7 -0
- package/src/service/storage/factory.ts +2 -2
- package/src/utils/config.ts +61 -8
- package/src/utils/credentials.ts +50 -0
- package/bin/contextlake-openclaw.js +0 -5
- package/dist/src/skills/las-data-profiler/register.d.ts +0 -1
- package/dist/src/skills/las-data-profiler/register.js +0 -19
- package/src/service/embedding/local.ts +0 -118
- package/src/skills/las-data-profiler/register.ts +0 -19
|
@@ -1 +1,34 @@
|
|
|
1
|
-
export
|
|
1
|
+
export interface EmbeddingConfig {
|
|
2
|
+
provider: string;
|
|
3
|
+
model_name: string;
|
|
4
|
+
api_key?: string;
|
|
5
|
+
api_base?: string;
|
|
6
|
+
dimensions?: number;
|
|
7
|
+
encoding_format?: string;
|
|
8
|
+
}
|
|
9
|
+
export interface MetadataStorageConfig {
|
|
10
|
+
type: 'local' | 'remote';
|
|
11
|
+
lancedb_uri?: string;
|
|
12
|
+
embedding?: EmbeddingConfig;
|
|
13
|
+
}
|
|
14
|
+
export interface FileStorageConfig {
|
|
15
|
+
type: 'local' | 'tos';
|
|
16
|
+
local_base_dir?: string;
|
|
17
|
+
tos?: {
|
|
18
|
+
access_key?: string;
|
|
19
|
+
secret_key?: string;
|
|
20
|
+
region?: string;
|
|
21
|
+
path?: string;
|
|
22
|
+
endpoint?: string;
|
|
23
|
+
sts_token?: string;
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
export interface StoragePolicyConfig {
|
|
27
|
+
max_inline_size_kb?: number;
|
|
28
|
+
}
|
|
29
|
+
export interface ContextLakeConfig {
|
|
30
|
+
metadata_storage?: MetadataStorageConfig;
|
|
31
|
+
file_storage?: FileStorageConfig;
|
|
32
|
+
storage_policy?: StoragePolicyConfig;
|
|
33
|
+
}
|
|
34
|
+
export declare function getPluginConfig(ctx: any): ContextLakeConfig;
|
package/dist/src/utils/config.js
CHANGED
|
@@ -1,16 +1,29 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.getPluginConfig = getPluginConfig;
|
|
4
|
+
const credentials_1 = require("./credentials");
|
|
4
5
|
function getPluginConfig(ctx) {
|
|
6
|
+
const creds = (0, credentials_1.loadCredentials)();
|
|
5
7
|
return ctx.config?.plugins?.entries?.['contextlake-openclaw']?.config || {
|
|
6
8
|
metadata_storage: {
|
|
7
9
|
type: 'local',
|
|
8
10
|
lancedb_uri: require('path').join(require('os').homedir(), '.openclaw', 'contextlake', 'data'),
|
|
9
11
|
embedding: {
|
|
10
|
-
provider: '
|
|
11
|
-
model_name: '
|
|
12
|
+
provider: 'las',
|
|
13
|
+
model_name: 'doubao-embedding-vision-250615',
|
|
14
|
+
api_key: process.env.LAS_API_KEY || creds.LAS_API_KEY,
|
|
15
|
+
api_base: process.env.LAS_BASE_URL || 'https://operator.las.cn-beijing.volces.com',
|
|
16
|
+
dimensions: 2048,
|
|
17
|
+
encoding_format: 'float'
|
|
12
18
|
}
|
|
13
19
|
},
|
|
14
|
-
file_storage: {
|
|
20
|
+
file_storage: {
|
|
21
|
+
type: 'local',
|
|
22
|
+
local_base_dir: require('path').join(require('os').homedir(), '.openclaw', 'contextlake', 'files'),
|
|
23
|
+
tos: {
|
|
24
|
+
access_key: process.env.VOLCENGINE_ACCESS_KEY || creds.VOLCENGINE_ACCESS_KEY,
|
|
25
|
+
secret_key: process.env.VOLCENGINE_SECRET_KEY || creds.VOLCENGINE_SECRET_KEY
|
|
26
|
+
}
|
|
27
|
+
}
|
|
15
28
|
};
|
|
16
29
|
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export interface Credentials {
|
|
2
|
+
LAS_API_KEY?: string;
|
|
3
|
+
VOLCENGINE_ACCESS_KEY?: string;
|
|
4
|
+
VOLCENGINE_SECRET_KEY?: string;
|
|
5
|
+
}
|
|
6
|
+
export declare function loadCredentials(): Credentials;
|
|
7
|
+
export declare function saveCredentials(creds: Credentials): void;
|
|
8
|
+
export declare function promptForInput(promptText: string, defaultVal?: string): Promise<string>;
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.loadCredentials = loadCredentials;
|
|
37
|
+
exports.saveCredentials = saveCredentials;
|
|
38
|
+
exports.promptForInput = promptForInput;
|
|
39
|
+
const fs = __importStar(require("fs"));
|
|
40
|
+
const path = __importStar(require("path"));
|
|
41
|
+
const os = __importStar(require("os"));
|
|
42
|
+
const readline = __importStar(require("readline"));
|
|
43
|
+
const CONFIG_DIR = path.join(os.homedir(), '.openclaw', 'contextlake');
|
|
44
|
+
const CONFIG_FILE = path.join(CONFIG_DIR, 'credentials.json');
|
|
45
|
+
function loadCredentials() {
|
|
46
|
+
if (fs.existsSync(CONFIG_FILE)) {
|
|
47
|
+
try {
|
|
48
|
+
const content = fs.readFileSync(CONFIG_FILE, 'utf-8');
|
|
49
|
+
return JSON.parse(content);
|
|
50
|
+
}
|
|
51
|
+
catch (e) {
|
|
52
|
+
// ignore
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return {};
|
|
56
|
+
}
|
|
57
|
+
function saveCredentials(creds) {
|
|
58
|
+
if (!fs.existsSync(CONFIG_DIR)) {
|
|
59
|
+
fs.mkdirSync(CONFIG_DIR, { recursive: true });
|
|
60
|
+
}
|
|
61
|
+
const current = loadCredentials();
|
|
62
|
+
const updated = { ...current, ...creds };
|
|
63
|
+
fs.writeFileSync(CONFIG_FILE, JSON.stringify(updated, null, 2), { mode: 0o600 });
|
|
64
|
+
}
|
|
65
|
+
async function promptForInput(promptText, defaultVal) {
|
|
66
|
+
const rl = readline.createInterface({
|
|
67
|
+
input: process.stdin,
|
|
68
|
+
output: process.stdout,
|
|
69
|
+
});
|
|
70
|
+
const displayPrompt = defaultVal ? `${promptText} [${defaultVal}]: ` : `${promptText}: `;
|
|
71
|
+
return new Promise((resolve) => {
|
|
72
|
+
rl.question(displayPrompt, (answer) => {
|
|
73
|
+
rl.close();
|
|
74
|
+
resolve(answer.trim() || defaultVal || '');
|
|
75
|
+
});
|
|
76
|
+
});
|
|
77
|
+
}
|
package/index.ts
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
// @ts-ignore
|
|
2
|
-
import {
|
|
2
|
+
import type { OpenClawPluginApi, PluginLogger } from 'openclaw/plugin-sdk';
|
|
3
3
|
import { registerAll } from './src/commands';
|
|
4
4
|
|
|
5
5
|
const plugin = {
|
|
6
6
|
id: 'contextlake-openclaw',
|
|
7
7
|
name: 'ContextLake',
|
|
8
|
-
version: '1.
|
|
8
|
+
version: '1.0.2',
|
|
9
9
|
description: 'A lightweight knowledge base plugin for OpenClaw using LanceDB and TOS, with data profiling support',
|
|
10
10
|
configSchema: {
|
|
11
11
|
type: 'object',
|
|
@@ -57,12 +57,12 @@ const plugin = {
|
|
|
57
57
|
}
|
|
58
58
|
}
|
|
59
59
|
},
|
|
60
|
-
register(ctx:
|
|
61
|
-
const logger = ctx.logger || {
|
|
62
|
-
info: (msg: string
|
|
63
|
-
warn: (msg: string
|
|
64
|
-
error: (msg: string
|
|
65
|
-
debug: (msg: string
|
|
60
|
+
register(ctx: OpenClawPluginApi) {
|
|
61
|
+
const logger: PluginLogger = ctx.logger || {
|
|
62
|
+
info: (msg: string) => console.log(msg),
|
|
63
|
+
warn: (msg: string) => console.warn(msg),
|
|
64
|
+
error: (msg: string) => console.error(msg),
|
|
65
|
+
debug: (msg: string) => console.debug(msg),
|
|
66
66
|
};
|
|
67
67
|
|
|
68
68
|
// Add logging
|
package/openclaw.plugin.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"id": "contextlake-openclaw",
|
|
3
3
|
"name": "ContextLake",
|
|
4
|
-
"version": "1.
|
|
4
|
+
"version": "1.0.3",
|
|
5
5
|
"description": "A lightweight knowledge base plugin for OpenClaw using LanceDB and TOS, with data profiling support",
|
|
6
6
|
"skills": ["./src/skills"],
|
|
7
7
|
"configSchema": {
|
package/package.json
CHANGED
|
@@ -1,31 +1,32 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@byted-las/contextlake-openclaw",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.3",
|
|
4
4
|
"description": "ContextLake OpenClaw Plugin for managing knowledge base",
|
|
5
5
|
"main": "index.ts",
|
|
6
6
|
"files": [
|
|
7
7
|
"dist",
|
|
8
|
-
"bin",
|
|
9
8
|
"index.ts",
|
|
10
9
|
"src",
|
|
11
10
|
"openclaw.plugin.json"
|
|
12
11
|
],
|
|
13
|
-
"bin": {
|
|
14
|
-
"contextlake-openclaw": "./bin/contextlake-openclaw.js"
|
|
15
|
-
},
|
|
16
12
|
"openclaw": {
|
|
17
13
|
"extensions": [
|
|
18
14
|
"./dist/index.js"
|
|
19
15
|
]
|
|
20
16
|
},
|
|
21
17
|
"scripts": {
|
|
22
|
-
"build": "tsc",
|
|
18
|
+
"build": "tsc && npm run copy-assets",
|
|
19
|
+
"copy-assets": "cp -r src/skills/*/*.md dist/src/skills/ 2>/dev/null || true && for dir in src/skills/*; do if [ -d \"$dir\" ]; then target=\"dist/$dir\"; mkdir -p \"$target\"; cp \"$dir\"/*.md \"$target\"/ 2>/dev/null || true; fi; done && mkdir -p dist/src/lib/scripts && cp src/lib/scripts/*.py dist/src/lib/scripts/ 2>/dev/null || true",
|
|
23
20
|
"test": "vitest --reporter verbose",
|
|
24
21
|
"test:local": "npx ts-node scripts/local-test.ts",
|
|
25
22
|
"test:profiler": "npx ts-node scripts/local-profiler-test.ts",
|
|
26
23
|
"cli": "npx ts-node scripts/cli.ts"
|
|
27
24
|
},
|
|
28
|
-
"keywords": [
|
|
25
|
+
"keywords": [
|
|
26
|
+
"openclaw",
|
|
27
|
+
"contextlake",
|
|
28
|
+
"plugin"
|
|
29
|
+
],
|
|
29
30
|
"author": "byted-las",
|
|
30
31
|
"license": "ISC",
|
|
31
32
|
"engines": {
|
package/src/client/lancedb.ts
CHANGED
|
@@ -38,27 +38,27 @@ export class ContextLakeLanceDBClient {
|
|
|
38
38
|
async getTable(tableName: string = 'documents', dim: number = 0) {
|
|
39
39
|
if (this.table) return this.table;
|
|
40
40
|
await this.connect();
|
|
41
|
-
|
|
41
|
+
|
|
42
42
|
const tableNames = await this.db!.tableNames();
|
|
43
43
|
if (tableNames.includes(tableName)) {
|
|
44
44
|
this.table = await this.db!.openTable(tableName);
|
|
45
45
|
} else {
|
|
46
46
|
if (dim <= 0) {
|
|
47
47
|
// Fallback: use embedding provider to infer dimension only if needed
|
|
48
|
-
const dummyVec = await this.embeddingProvider.generateEmbedding(
|
|
48
|
+
const dummyVec = await this.embeddingProvider.generateEmbedding('init');
|
|
49
49
|
dim = dummyVec.length;
|
|
50
50
|
}
|
|
51
|
-
|
|
51
|
+
|
|
52
52
|
// @ts-ignore
|
|
53
53
|
this.table = await this.db!.createTable(tableName, [
|
|
54
|
-
{
|
|
55
|
-
id: 'schema_init',
|
|
56
|
-
vector: Array(dim).fill(0),
|
|
57
|
-
text: '',
|
|
58
|
-
source: '',
|
|
54
|
+
{
|
|
55
|
+
id: 'schema_init',
|
|
56
|
+
vector: Array(dim).fill(0),
|
|
57
|
+
text: '',
|
|
58
|
+
source: '',
|
|
59
59
|
file_type: '',
|
|
60
60
|
storage_type: '',
|
|
61
|
-
url: '',
|
|
61
|
+
url: '',
|
|
62
62
|
metadata: '{}',
|
|
63
63
|
created_at: 0,
|
|
64
64
|
binary_data: Buffer.from('')
|
|
@@ -76,27 +76,38 @@ export class ContextLakeLanceDBClient {
|
|
|
76
76
|
}
|
|
77
77
|
|
|
78
78
|
async search(query: string, limit: number = 5, filter?: string) {
|
|
79
|
-
const
|
|
79
|
+
const normalizedLimit = Number.isFinite(limit) ? Math.max(1, Math.floor(limit)) : 5;
|
|
80
80
|
const table = await this.getTable();
|
|
81
|
+
|
|
82
|
+
if (!query || !query.trim()) {
|
|
83
|
+
let fallbackQuery = table.query().limit(normalizedLimit);
|
|
84
|
+
if (filter) {
|
|
85
|
+
fallbackQuery = fallbackQuery.where(filter);
|
|
86
|
+
}
|
|
87
|
+
return await fallbackQuery.toArray();
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const vector = await this.embeddingProvider.generateMultimodalEmbedding!([{ type: 'text', text: query }]);
|
|
81
91
|
// @ts-ignore
|
|
82
|
-
let search = table.vectorSearch(vector).limit(
|
|
92
|
+
let search = table.vectorSearch(vector).limit(normalizedLimit);
|
|
83
93
|
if (filter) {
|
|
84
94
|
search = search.where(filter);
|
|
85
95
|
}
|
|
86
96
|
return await search.toArray();
|
|
87
97
|
}
|
|
88
|
-
|
|
98
|
+
|
|
89
99
|
async delete(filter: string) {
|
|
90
|
-
|
|
91
|
-
|
|
100
|
+
const table = await this.getTable();
|
|
101
|
+
await table.delete(filter);
|
|
92
102
|
}
|
|
93
|
-
|
|
103
|
+
|
|
94
104
|
async list(limit: number = 100, filter?: string) {
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
105
|
+
const normalizedLimit = Number.isFinite(limit) ? Math.max(1, Math.floor(limit)) : 100;
|
|
106
|
+
const table = await this.getTable();
|
|
107
|
+
let query = table.query().limit(normalizedLimit);
|
|
108
|
+
if (filter) {
|
|
109
|
+
query = query.where(filter);
|
|
110
|
+
}
|
|
111
|
+
return await query.toArray();
|
|
101
112
|
}
|
|
102
113
|
}
|
package/src/commands/cli.ts
CHANGED
|
@@ -1,20 +1,83 @@
|
|
|
1
1
|
// @ts-ignore
|
|
2
|
-
import {
|
|
3
|
-
import { ingestAssets } from '../lib/actions/ingest';
|
|
2
|
+
import { ingestSource } from '../lib/actions/ingest';
|
|
4
3
|
import { retrieveAssets } from '../lib/actions/retrieve';
|
|
5
4
|
import { listAssets, deleteAssets } from '../lib/actions/manage';
|
|
5
|
+
import { connectDataSource, ConnectParams } from '../lib/actions/profiler';
|
|
6
|
+
import { loadCredentials, saveCredentials, promptForInput } from '../utils/credentials';
|
|
7
|
+
import { ContextLakeConfig } from '../utils/config';
|
|
6
8
|
|
|
7
|
-
|
|
9
|
+
function parseOptionalInt(value: any, fallback: number): number {
|
|
10
|
+
const parsed = Number.parseInt(String(value), 10);
|
|
11
|
+
return Number.isFinite(parsed) ? parsed : fallback;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function parseMetadata(metadata: any): Record<string, any> {
|
|
15
|
+
if (!metadata) {
|
|
16
|
+
return {};
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
if (typeof metadata === 'object') {
|
|
20
|
+
return metadata;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
if (typeof metadata !== 'string') {
|
|
24
|
+
throw new Error('metadata must be a JSON object or JSON string');
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
try {
|
|
28
|
+
const parsed = JSON.parse(metadata);
|
|
29
|
+
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
|
30
|
+
throw new Error('metadata must be a JSON object');
|
|
31
|
+
}
|
|
32
|
+
return parsed;
|
|
33
|
+
} catch (error: any) {
|
|
34
|
+
throw new Error(`Invalid metadata JSON: ${error.message}`);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function getCliCommands(pluginConfig: ContextLakeConfig, logger: any) {
|
|
8
39
|
return {
|
|
9
|
-
|
|
10
|
-
logger.info(`[${new Date().toISOString()}] [ContextLake] CLI
|
|
40
|
+
connectAction: async (datasource_name: string, options: any) => {
|
|
41
|
+
logger.info(`[${new Date().toISOString()}] [ContextLake] CLI connect started`, { datasource_name, options });
|
|
11
42
|
try {
|
|
12
|
-
const
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
43
|
+
const params: ConnectParams = {
|
|
44
|
+
datasource_name,
|
|
45
|
+
vendor: options.vendor,
|
|
46
|
+
endpoint: options.endpoint,
|
|
47
|
+
access_key: options.ak,
|
|
48
|
+
secret_key: options.sk,
|
|
49
|
+
region: options.region,
|
|
50
|
+
bucket: options.bucket,
|
|
51
|
+
prefix: options.prefix,
|
|
52
|
+
sample_rows: parseInt(options.sampleRows),
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
// eslint-disable-next-line no-console
|
|
56
|
+
console.log(`[contextlake connect] Connecting to datasource "${datasource_name}"...`);
|
|
57
|
+
// eslint-disable-next-line no-console
|
|
58
|
+
console.log(` vendor: ${params.vendor}`);
|
|
59
|
+
// eslint-disable-next-line no-console
|
|
60
|
+
console.log(` bucket: ${params.bucket}`);
|
|
61
|
+
// eslint-disable-next-line no-console
|
|
62
|
+
console.log(` prefix: ${params.prefix}`);
|
|
63
|
+
|
|
64
|
+
const result = await connectDataSource(params);
|
|
65
|
+
// eslint-disable-next-line no-console
|
|
66
|
+
console.log(JSON.stringify(result, null, 2));
|
|
67
|
+
logger.info(`[${new Date().toISOString()}] [ContextLake] CLI connect success`);
|
|
68
|
+
} catch (e: any) {
|
|
69
|
+
// eslint-disable-next-line no-console
|
|
70
|
+
console.error('Error:', e.message);
|
|
71
|
+
logger.error(`[${new Date().toISOString()}] [ContextLake] CLI connect failed`, { error: e.message, stack: e.stack });
|
|
72
|
+
process.exitCode = 1;
|
|
73
|
+
}
|
|
74
|
+
},
|
|
75
|
+
|
|
76
|
+
ingestAction: async (datasource_name: string) => {
|
|
77
|
+
logger.info(`[${new Date().toISOString()}] [ContextLake] CLI ingest started`, { datasource_name });
|
|
78
|
+
try {
|
|
79
|
+
const result = await ingestSource({
|
|
80
|
+
datasource_name
|
|
18
81
|
}, pluginConfig, logger);
|
|
19
82
|
// eslint-disable-next-line no-console
|
|
20
83
|
console.log(JSON.stringify(result, null, 2));
|
|
@@ -30,7 +93,7 @@ export function getCliCommands(pluginConfig: any, logger: any) {
|
|
|
30
93
|
try {
|
|
31
94
|
const result = await retrieveAssets({
|
|
32
95
|
query,
|
|
33
|
-
top_k:
|
|
96
|
+
top_k: parseOptionalInt(options.topK, 5),
|
|
34
97
|
filter: options.filter,
|
|
35
98
|
include_binary: options.binary
|
|
36
99
|
}, pluginConfig, logger);
|
|
@@ -47,7 +110,7 @@ export function getCliCommands(pluginConfig: any, logger: any) {
|
|
|
47
110
|
logger.info(`[${new Date().toISOString()}] [ContextLake] CLI list started`, { options });
|
|
48
111
|
try {
|
|
49
112
|
const result = await listAssets({
|
|
50
|
-
limit:
|
|
113
|
+
limit: parseOptionalInt(options.limit, 100)
|
|
51
114
|
}, pluginConfig, logger);
|
|
52
115
|
// eslint-disable-next-line no-console
|
|
53
116
|
console.log(JSON.stringify(result, null, 2));
|
|
@@ -72,6 +135,35 @@ export function getCliCommands(pluginConfig: any, logger: any) {
|
|
|
72
135
|
console.error('Error:', e.message);
|
|
73
136
|
logger.error(`[${new Date().toISOString()}] [ContextLake] CLI delete failed`, { error: e.message, stack: e.stack });
|
|
74
137
|
}
|
|
138
|
+
},
|
|
139
|
+
|
|
140
|
+
onboardAction: async () => {
|
|
141
|
+
logger.info(`[${new Date().toISOString()}] [ContextLake] CLI onboard started`);
|
|
142
|
+
try {
|
|
143
|
+
const currentCreds = loadCredentials();
|
|
144
|
+
// eslint-disable-next-line no-console
|
|
145
|
+
console.log('Welcome to ContextLake Onboarding!');
|
|
146
|
+
// eslint-disable-next-line no-console
|
|
147
|
+
console.log('Please provide your credentials below. Press enter to keep the current value.');
|
|
148
|
+
|
|
149
|
+
const lasApiKey = await promptForInput('LAS_API_KEY', currentCreds.LAS_API_KEY);
|
|
150
|
+
const volcengineAccessKey = await promptForInput('VOLCENGINE_ACCESS_KEY', currentCreds.VOLCENGINE_ACCESS_KEY);
|
|
151
|
+
const volcengineSecretKey = await promptForInput('VOLCENGINE_SECRET_KEY', currentCreds.VOLCENGINE_SECRET_KEY);
|
|
152
|
+
|
|
153
|
+
const newCreds = {
|
|
154
|
+
LAS_API_KEY: lasApiKey,
|
|
155
|
+
VOLCENGINE_ACCESS_KEY: volcengineAccessKey,
|
|
156
|
+
VOLCENGINE_SECRET_KEY: volcengineSecretKey
|
|
157
|
+
};
|
|
158
|
+
|
|
159
|
+
saveCredentials(newCreds);
|
|
160
|
+
// eslint-disable-next-line no-console
|
|
161
|
+
console.log('Credentials saved successfully!');
|
|
162
|
+
logger.info(`[${new Date().toISOString()}] [ContextLake] CLI onboard success`);
|
|
163
|
+
} catch (e: any) {
|
|
164
|
+
console.error('Error during onboarding:', e.message);
|
|
165
|
+
logger.error(`[${new Date().toISOString()}] [ContextLake] CLI onboard failed`, { error: e.message, stack: e.stack });
|
|
166
|
+
}
|
|
75
167
|
}
|
|
76
168
|
};
|
|
77
169
|
}
|
package/src/commands/index.ts
CHANGED
|
@@ -2,29 +2,41 @@ import { getAgentTools } from './tools';
|
|
|
2
2
|
import { getCliCommands } from './cli';
|
|
3
3
|
import { getSlashCommands } from './slashcmd';
|
|
4
4
|
import { getPluginConfig } from '../utils/config';
|
|
5
|
-
|
|
5
|
+
// @ts-ignore
|
|
6
|
+
import type { OpenClawPluginApi, PluginLogger, AnyAgentTool } from 'openclaw/plugin-sdk';
|
|
6
7
|
|
|
7
|
-
export function registerAll(ctx:
|
|
8
|
+
export function registerAll(ctx: OpenClawPluginApi, logger: PluginLogger) {
|
|
8
9
|
const pluginConfig = getPluginConfig(ctx);
|
|
9
10
|
|
|
10
11
|
// Register Agent Tools
|
|
11
12
|
try {
|
|
12
13
|
const tools = getAgentTools(pluginConfig, logger);
|
|
13
14
|
|
|
14
|
-
ctx.registerTool(tools.ingestTool);
|
|
15
|
+
ctx.registerTool(tools.ingestTool );
|
|
15
16
|
logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${tools.ingestTool.name}`);
|
|
16
17
|
|
|
17
|
-
ctx.registerTool(tools.retrieveTool);
|
|
18
|
+
ctx.registerTool(tools.retrieveTool );
|
|
18
19
|
logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${tools.retrieveTool.name}`);
|
|
19
20
|
|
|
20
|
-
ctx.registerTool(tools.listTool);
|
|
21
|
+
ctx.registerTool(tools.listTool );
|
|
21
22
|
logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${tools.listTool.name}`);
|
|
22
23
|
|
|
23
|
-
ctx.registerTool(tools.deleteTool);
|
|
24
|
+
ctx.registerTool(tools.deleteTool );
|
|
24
25
|
logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${tools.deleteTool.name}`);
|
|
25
26
|
|
|
27
|
+
ctx.registerTool(tools.lasDataProfilerTool );
|
|
28
|
+
logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${tools.lasDataProfilerTool.name}`);
|
|
29
|
+
|
|
30
|
+
ctx.registerTool(tools.listDatasourceTool );
|
|
31
|
+
logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${tools.listDatasourceTool.name}`);
|
|
32
|
+
|
|
33
|
+
for (const lasTool of tools.lasTools) {
|
|
34
|
+
ctx.registerTool(lasTool);
|
|
35
|
+
logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${lasTool.name}`);
|
|
36
|
+
}
|
|
37
|
+
|
|
26
38
|
} catch (error: any) {
|
|
27
|
-
logger.error(`[${new Date().toISOString()}] [ContextLake] Error registering agent tools: ${error.message}
|
|
39
|
+
logger.error(`[${new Date().toISOString()}] [ContextLake] Error registering agent tools: ${error.message}${error.stack ? '\\n' + error.stack : ''}`);
|
|
28
40
|
throw error;
|
|
29
41
|
}
|
|
30
42
|
|
|
@@ -49,39 +61,11 @@ export function registerAll(ctx: any, logger: any) {
|
|
|
49
61
|
.requiredOption('--bucket <bucket>', 'Bucket name (or local root directory for local vendor)')
|
|
50
62
|
.requiredOption('--prefix <prefix>', 'Path prefix to limit scan scope')
|
|
51
63
|
.option('--sample-rows <number>', 'Number of rows to sample per structured file', '100')
|
|
52
|
-
.action(
|
|
53
|
-
try {
|
|
54
|
-
const params: ConnectParams = {
|
|
55
|
-
datasource_name,
|
|
56
|
-
vendor: options.vendor,
|
|
57
|
-
endpoint: options.endpoint,
|
|
58
|
-
access_key: options.ak,
|
|
59
|
-
secret_key: options.sk,
|
|
60
|
-
region: options.region,
|
|
61
|
-
bucket: options.bucket,
|
|
62
|
-
prefix: options.prefix,
|
|
63
|
-
sample_rows: parseInt(options.sampleRows),
|
|
64
|
-
};
|
|
65
|
-
|
|
66
|
-
console.log(`[contextlake connect] Connecting to datasource "${datasource_name}"...`);
|
|
67
|
-
console.log(` vendor: ${params.vendor}`);
|
|
68
|
-
console.log(` bucket: ${params.bucket}`);
|
|
69
|
-
console.log(` prefix: ${params.prefix}`);
|
|
70
|
-
|
|
71
|
-
const result = await connectDataSource(params);
|
|
72
|
-
console.log(JSON.stringify(result, null, 2));
|
|
73
|
-
} catch (e: any) {
|
|
74
|
-
console.error('Error:', e.message);
|
|
75
|
-
process.exitCode = 1;
|
|
76
|
-
}
|
|
77
|
-
});
|
|
64
|
+
.action(commands.connectAction);
|
|
78
65
|
|
|
79
66
|
// Ingest
|
|
80
|
-
contextlake.command('ingest <
|
|
81
|
-
.description('
|
|
82
|
-
.option('-c, --chunk-size <number>', 'Chunk size for text splitting', '500')
|
|
83
|
-
.option('-o, --overlap <number>', 'Chunk overlap size', '50')
|
|
84
|
-
.option('-m, --metadata <json>', 'JSON metadata to attach to the documents')
|
|
67
|
+
contextlake.command('ingest <datasource_name>')
|
|
68
|
+
.description('Process and ingest all files from a connected data source into the knowledge base')
|
|
85
69
|
.action(commands.ingestAction);
|
|
86
70
|
|
|
87
71
|
// Search
|
|
@@ -104,11 +88,16 @@ export function registerAll(ctx: any, logger: any) {
|
|
|
104
88
|
.option('--ids <ids...>', 'List of specific file IDs to delete')
|
|
105
89
|
.option('-f, --filter <string>', 'Filter string to match documents for deletion')
|
|
106
90
|
.action(commands.deleteAction);
|
|
91
|
+
|
|
92
|
+
// Onboard
|
|
93
|
+
contextlake.command('onboard')
|
|
94
|
+
.description('Configure credentials for ContextLake')
|
|
95
|
+
.action(commands.onboardAction);
|
|
107
96
|
|
|
108
97
|
}, { commands: ['contextlake'] });
|
|
109
98
|
logger.info(`[${new Date().toISOString()}] [ContextLake] CLI commands registered`);
|
|
110
99
|
} catch (error: any) {
|
|
111
|
-
logger.error(`[${new Date().toISOString()}] [ContextLake] Error registering CLI commands: ${error.message}
|
|
100
|
+
logger.error(`[${new Date().toISOString()}] [ContextLake] Error registering CLI commands: ${error.message}${error.stack ? '\\n' + error.stack : ''}`);
|
|
112
101
|
throw error;
|
|
113
102
|
}
|
|
114
103
|
|
|
@@ -123,7 +112,7 @@ export function registerAll(ctx: any, logger: any) {
|
|
|
123
112
|
|
|
124
113
|
ctx.registerCommand({
|
|
125
114
|
name: 'contextlake-ingest',
|
|
126
|
-
description: '
|
|
115
|
+
description: 'Process and ingest all files from a connected data source (usage: /contextlake-ingest <datasource_name>)',
|
|
127
116
|
acceptsArgs: true,
|
|
128
117
|
handler: slashCommands.ingestHandler
|
|
129
118
|
});
|
|
@@ -148,9 +137,23 @@ export function registerAll(ctx: any, logger: any) {
|
|
|
148
137
|
acceptsArgs: true,
|
|
149
138
|
handler: slashCommands.deleteHandler
|
|
150
139
|
});
|
|
140
|
+
|
|
141
|
+
ctx.registerCommand({
|
|
142
|
+
name: 'contextlake-profiler',
|
|
143
|
+
description: 'Connect to a data source and profile its structure (usage: /contextlake-profiler <datasource_name> <vendor> <bucket> <prefix>)',
|
|
144
|
+
acceptsArgs: true,
|
|
145
|
+
handler: slashCommands.profilerHandler
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
ctx.registerCommand({
|
|
149
|
+
name: 'contextlake-list-datasource',
|
|
150
|
+
description: 'List all connected and profiled data sources (usage: /contextlake-list-datasource)',
|
|
151
|
+
acceptsArgs: false,
|
|
152
|
+
handler: slashCommands.listDatasourceHandler
|
|
153
|
+
});
|
|
151
154
|
|
|
152
155
|
logger.info(`[${new Date().toISOString()}] [ContextLake] Slash commands registered`);
|
|
153
156
|
} catch (error: any) {
|
|
154
|
-
logger.error(`[${new Date().toISOString()}] [ContextLake] Error registering Slash commands: ${error.message}
|
|
157
|
+
logger.error(`[${new Date().toISOString()}] [ContextLake] Error registering Slash commands: ${error.message}${error.stack ? '\\n' + error.stack : ''}`);
|
|
155
158
|
}
|
|
156
|
-
}
|
|
159
|
+
}
|