@byted-las/contextlake-openclaw 1.0.2 → 1.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/dist/src/client/lancedb.js +1 -1
  2. package/dist/src/commands/cli.d.ts +2 -1
  3. package/dist/src/commands/cli.js +31 -8
  4. package/dist/src/commands/index.js +25 -6
  5. package/dist/src/commands/slashcmd.d.ts +6 -0
  6. package/dist/src/commands/slashcmd.js +90 -6
  7. package/dist/src/commands/tools.d.ts +2 -0
  8. package/dist/src/commands/tools.js +44 -37
  9. package/dist/src/lib/actions/ingest-source.d.ts +15 -0
  10. package/dist/src/lib/actions/ingest-source.js +193 -0
  11. package/dist/src/lib/actions/ingest.d.ts +13 -7
  12. package/dist/src/lib/actions/ingest.js +133 -58
  13. package/dist/src/lib/actions/las-api.d.ts +13 -0
  14. package/dist/src/lib/actions/las-api.js +105 -0
  15. package/dist/src/lib/actions/las-tools.d.ts +3 -0
  16. package/dist/src/lib/actions/las-tools.js +194 -0
  17. package/dist/src/lib/actions/las.d.ts +64 -0
  18. package/dist/src/lib/actions/las.js +72 -0
  19. package/dist/src/lib/actions/profiler.d.ts +3 -0
  20. package/dist/src/lib/actions/profiler.js +17 -1
  21. package/dist/src/lib/actions/retrieve.js +2 -8
  22. package/dist/src/lib/scripts/s3_catalog.py +10 -1
  23. package/dist/src/service/embedding/factory.js +1 -10
  24. package/dist/src/service/embedding/interface.d.ts +5 -0
  25. package/dist/src/service/embedding/remote.d.ts +1 -0
  26. package/dist/src/service/embedding/remote.js +31 -0
  27. package/dist/src/service/metadata/interface.d.ts +1 -0
  28. package/dist/src/service/metadata/local.d.ts +1 -0
  29. package/dist/src/service/metadata/local.js +6 -0
  30. package/dist/src/utils/config.js +11 -2
  31. package/dist/src/utils/credentials.d.ts +8 -0
  32. package/dist/src/utils/credentials.js +77 -0
  33. package/openclaw.plugin.json +1 -1
  34. package/package.json +1 -5
  35. package/src/client/lancedb.ts +1 -1
  36. package/src/commands/cli.ts +35 -9
  37. package/src/commands/index.ts +30 -6
  38. package/src/commands/slashcmd.ts +67 -7
  39. package/src/commands/tools.ts +49 -41
  40. package/src/lib/actions/ingest.ts +151 -71
  41. package/src/lib/actions/las-api.ts +119 -0
  42. package/src/lib/actions/las-tools.ts +196 -0
  43. package/src/lib/actions/profiler.ts +18 -1
  44. package/src/lib/actions/retrieve.ts +2 -10
  45. package/src/lib/scripts/s3_catalog.py +10 -1
  46. package/src/service/embedding/factory.ts +1 -8
  47. package/src/service/embedding/interface.ts +6 -0
  48. package/src/service/embedding/remote.ts +36 -0
  49. package/src/service/metadata/interface.ts +1 -0
  50. package/src/service/metadata/local.ts +7 -0
  51. package/src/utils/config.ts +13 -2
  52. package/src/utils/credentials.ts +50 -0
  53. package/bin/contextlake-openclaw.js +0 -5
  54. package/src/service/embedding/local.ts +0 -121
@@ -0,0 +1,72 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.lasPdfParseDoubao = lasPdfParseDoubao;
4
+ exports.lasLongVideoUnderstand = lasLongVideoUnderstand;
5
+ exports.lasBareImageTextEmbedding = lasBareImageTextEmbedding;
6
+ exports.lasSeed20 = lasSeed20;
7
+ exports.lasAsrPro = lasAsrPro;
8
+ exports.lasAudioExtractAndSplit = lasAudioExtractAndSplit;
9
+ function getLASConfig(config) {
10
+ // Attempt to get from env vars or config
11
+ const endpoint = process.env.LAS_ENDPOINT || (config?.las)?.endpoint;
12
+ const apiKey = process.env.LAS_API_KEY || (config?.las)?.api_key;
13
+ if (!endpoint || !apiKey) {
14
+ throw new Error("LAS_ENDPOINT and LAS_API_KEY must be set in environment variables or config");
15
+ }
16
+ return { endpoint, apiKey };
17
+ }
18
+ async function lasFetch(path, payload, config) {
19
+ const { endpoint, apiKey } = getLASConfig(config);
20
+ const url = `${endpoint.replace(/\/$/, '')}${path}`;
21
+ const response = await fetch(url, {
22
+ method: 'POST',
23
+ headers: {
24
+ 'Content-Type': 'application/json',
25
+ 'Authorization': `Bearer ${apiKey}`
26
+ },
27
+ body: JSON.stringify(payload)
28
+ });
29
+ if (!response.ok) {
30
+ let errorText = await response.text().catch(() => '');
31
+ throw new Error(`LAS API Error: ${response.status} ${response.statusText} - ${errorText}`);
32
+ }
33
+ return await response.json();
34
+ }
35
+ async function lasPdfParseDoubao(params, config) {
36
+ return lasFetch('/api/v1/submit', {
37
+ operator_id: 'las_pdf_parse_doubao',
38
+ operator_version: 'v1',
39
+ data: params
40
+ }, config);
41
+ }
42
+ async function lasLongVideoUnderstand(params, config) {
43
+ return lasFetch('/api/v1/submit', {
44
+ operator_id: 'las_long_video_understand',
45
+ operator_version: 'v1',
46
+ data: params
47
+ }, config);
48
+ }
49
+ async function lasBareImageTextEmbedding(params, config) {
50
+ return lasFetch('/api/v1/embeddings/multimodal', {
51
+ model: 'doubao-embedding-vision',
52
+ input: params.input,
53
+ encoding_format: params.encoding_format
54
+ }, config);
55
+ }
56
+ async function lasSeed20(params, config) {
57
+ return lasFetch('/api/v1/chat/completions', params, config);
58
+ }
59
+ async function lasAsrPro(params, config) {
60
+ return lasFetch('/api/v1/submit', {
61
+ operator_id: 'las_asr_pro',
62
+ operator_version: 'v1',
63
+ data: params
64
+ }, config);
65
+ }
66
+ async function lasAudioExtractAndSplit(params, config) {
67
+ return lasFetch('/api/v1/process', {
68
+ operator_id: 'las_audio_extract_and_split',
69
+ operator_version: 'v1',
70
+ data: params
71
+ }, config);
72
+ }
@@ -23,3 +23,6 @@ export interface ConnectResult {
23
23
  error?: string;
24
24
  }
25
25
  export declare function connectDataSource(params: ConnectParams, _ctx?: any): Promise<ConnectResult>;
26
+ export declare function listDataSources(_ctx?: any): Promise<{
27
+ datasources: string[];
28
+ }>;
@@ -34,6 +34,7 @@ var __importStar = (this && this.__importStar) || (function () {
34
34
  })();
35
35
  Object.defineProperty(exports, "__esModule", { value: true });
36
36
  exports.connectDataSource = connectDataSource;
37
+ exports.listDataSources = listDataSources;
37
38
  const path = __importStar(require("path"));
38
39
  const fs = __importStar(require("fs"));
39
40
  const os = __importStar(require("os"));
@@ -41,7 +42,7 @@ const child_process_1 = require("child_process");
41
42
  // ---------------------------------------------------------------------------
42
43
  // Constants
43
44
  // ---------------------------------------------------------------------------
44
- const BASE_DIR = path.join(os.homedir(), '.openclaw', 'las-data-profiler');
45
+ const BASE_DIR = path.join(os.homedir(), '.openclaw', 'contextlake', 'profiler');
45
46
  const PYTHON_DEPS = ['boto3', 'lancedb', 'pyarrow', 'pandas', 'Pillow', 'mutagen', 'pymupdf'];
46
47
  // ---------------------------------------------------------------------------
47
48
  // Helpers
@@ -229,3 +230,18 @@ async function connectDataSource(params, _ctx) {
229
230
  });
230
231
  });
231
232
  }
233
+ async function listDataSources(_ctx) {
234
+ try {
235
+ if (!fs.existsSync(BASE_DIR)) {
236
+ return { datasources: [] };
237
+ }
238
+ const entries = fs.readdirSync(BASE_DIR, { withFileTypes: true });
239
+ const datasources = entries
240
+ .filter(entry => entry.isDirectory())
241
+ .map(entry => entry.name);
242
+ return { datasources };
243
+ }
244
+ catch (error) {
245
+ throw new Error(`Failed to list data sources: ${error.message}`);
246
+ }
247
+ }
@@ -21,10 +21,7 @@ async function retrieveAssets(params, config, logger) {
21
21
  const results = await metadataProvider.search(params.query, params.top_k || 5, params.filter);
22
22
  if (params.include_binary) {
23
23
  const enrichedResults = await Promise.all(results.map(async (doc) => {
24
- const { binary_data, ...rest } = doc;
25
- if (rest.vector) {
26
- rest.vector = Array.from(rest.vector);
27
- }
24
+ const { binary_data, vector, ...rest } = doc;
28
25
  try {
29
26
  if (rest.storage_type === 'inline') {
30
27
  return {
@@ -53,10 +50,7 @@ async function retrieveAssets(params, config, logger) {
53
50
  return enrichedResults;
54
51
  }
55
52
  return results.map((doc) => {
56
- const { binary_data, ...rest } = doc;
57
- if (rest.vector) {
58
- rest.vector = Array.from(rest.vector);
59
- }
53
+ const { binary_data, vector, ...rest } = doc;
60
54
  const safeDoc = JSON.parse(JSON.stringify(rest));
61
55
  return safeDoc;
62
56
  });
@@ -523,10 +523,19 @@ def main():
523
523
  parser.add_argument('--region', default='')
524
524
  parser.add_argument('--bucket', required=True)
525
525
  parser.add_argument('--prefix', required=True)
526
- parser.add_argument('--db-path', default='./catalog_db')
526
+ parser.add_argument('--db-path', default=None, help='Path to LanceDB database. Defaults to ~/.openclaw/las-data-profiler/{datasource_name}/catalog_db if datasource_name is provided.')
527
+ parser.add_argument('--datasource-name', default='', help='Name of the datasource. Used to determine default db-path if not explicitly provided.')
527
528
  parser.add_argument('--sample-rows', type=int, default=100)
528
529
  args = parser.parse_args()
529
530
 
531
+ if not args.db_path:
532
+ if args.datasource_name:
533
+ import os
534
+ home_dir = os.path.expanduser('~')
535
+ args.db_path = os.path.join(home_dir, '.openclaw', 'las-data-profiler', args.datasource_name, 'catalog_db')
536
+ else:
537
+ args.db_path = './catalog_db'
538
+
530
539
  print(f"[las-data-profiler] vendor={args.vendor}, bucket={args.bucket}, prefix={args.prefix}")
531
540
  print(f"[las-data-profiler] db_path={args.db_path}")
532
541
 
@@ -1,16 +1,7 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.createEmbeddingProvider = createEmbeddingProvider;
4
- const local_1 = require("./local");
5
4
  const remote_1 = require("./remote");
6
5
  function createEmbeddingProvider(config) {
7
- if (config.provider === 'local') {
8
- return new local_1.LocalEmbeddingProvider(config);
9
- }
10
- else if (config.provider === 'openai' || config.provider === 'remote' || config.provider === 'las') {
11
- return new remote_1.RemoteEmbeddingProvider(config);
12
- }
13
- else {
14
- throw new Error(`Unsupported embedding provider: ${config.provider}`);
15
- }
6
+ return new remote_1.RemoteEmbeddingProvider(config);
16
7
  }
@@ -9,6 +9,11 @@ export interface EmbeddingProvider {
9
9
  * @param texts - Array of input texts
10
10
  */
11
11
  generateEmbeddings(texts: string[]): Promise<number[][]>;
12
+ /**
13
+ * Generate embedding for multimodal input (LAS specific)
14
+ * @param input - Multimodal input array
15
+ */
16
+ generateMultimodalEmbedding?(input: any[]): Promise<number[]>;
12
17
  }
13
18
  export interface EmbeddingConfig {
14
19
  provider: 'local' | 'remote' | 'openai' | 'las';
@@ -12,4 +12,5 @@ export declare class RemoteEmbeddingProvider implements EmbeddingProvider {
12
12
  generateEmbeddings(texts: string[]): Promise<number[][]>;
13
13
  private generateOpenAICompatibleEmbeddings;
14
14
  private generateLasEmbeddings;
15
+ generateMultimodalEmbedding(input: any[]): Promise<number[]>;
15
16
  }
@@ -108,5 +108,36 @@ class RemoteEmbeddingProvider {
108
108
  });
109
109
  return Promise.all(requests);
110
110
  }
111
+ async generateMultimodalEmbedding(input) {
112
+ if (this.mode !== 'las-multimodal') {
113
+ throw new Error('generateMultimodalEmbedding requires LAS multimodal provider');
114
+ }
115
+ const endpoint = this.apiBase.endsWith('/api/v1/embeddings/multimodal')
116
+ ? this.apiBase
117
+ : `${this.apiBase}/api/v1/embeddings/multimodal`;
118
+ const response = await fetch(endpoint, {
119
+ method: 'POST',
120
+ headers: {
121
+ 'Content-Type': 'application/json',
122
+ 'Authorization': `Bearer ${this.apiKey}`
123
+ },
124
+ body: JSON.stringify({
125
+ model: this.modelName,
126
+ encoding_format: this.encodingFormat,
127
+ ...(this.dimensions ? { dimensions: this.dimensions } : {}),
128
+ input
129
+ })
130
+ });
131
+ if (!response.ok) {
132
+ const error = await response.text();
133
+ throw new Error(`LAS embedding API error: ${response.status} ${error}`);
134
+ }
135
+ const data = await response.json();
136
+ const embedding = data?.data?.embedding;
137
+ if (!Array.isArray(embedding) || embedding.length === 0) {
138
+ throw new Error('Unexpected LAS embedding response format');
139
+ }
140
+ return embedding;
141
+ }
111
142
  }
112
143
  exports.RemoteEmbeddingProvider = RemoteEmbeddingProvider;
@@ -7,6 +7,7 @@ export interface MetadataProvider {
7
7
  list(limit?: number, filter?: string): Promise<DocumentSchema[]>;
8
8
  delete(filter: string): Promise<void>;
9
9
  generateEmbedding(text: string): Promise<number[]>;
10
+ generateMultimodalEmbedding?(input: any[]): Promise<number[]>;
10
11
  }
11
12
  export interface MetadataConfig {
12
13
  type: 'local' | 'remote';
@@ -10,4 +10,5 @@ export declare class LocalMetadataProvider implements MetadataProvider {
10
10
  list(limit?: number, filter?: string): Promise<DocumentSchema[]>;
11
11
  delete(filter: string): Promise<void>;
12
12
  generateEmbedding(text: string): Promise<number[]>;
13
+ generateMultimodalEmbedding(input: any[]): Promise<number[]>;
13
14
  }
@@ -45,5 +45,11 @@ class LocalMetadataProvider {
45
45
  async generateEmbedding(text) {
46
46
  return await this.embeddingProvider.generateEmbedding(text);
47
47
  }
48
+ async generateMultimodalEmbedding(input) {
49
+ if (this.embeddingProvider.generateMultimodalEmbedding) {
50
+ return await this.embeddingProvider.generateMultimodalEmbedding(input);
51
+ }
52
+ throw new Error('generateMultimodalEmbedding not supported by this embedding provider');
53
+ }
48
54
  }
49
55
  exports.LocalMetadataProvider = LocalMetadataProvider;
@@ -1,7 +1,9 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.getPluginConfig = getPluginConfig;
4
+ const credentials_1 = require("./credentials");
4
5
  function getPluginConfig(ctx) {
6
+ const creds = (0, credentials_1.loadCredentials)();
5
7
  return ctx.config?.plugins?.entries?.['contextlake-openclaw']?.config || {
6
8
  metadata_storage: {
7
9
  type: 'local',
@@ -9,12 +11,19 @@ function getPluginConfig(ctx) {
9
11
  embedding: {
10
12
  provider: 'las',
11
13
  model_name: 'doubao-embedding-vision-250615',
12
- api_key: process.env.LAS_API_KEY,
14
+ api_key: process.env.LAS_API_KEY || creds.LAS_API_KEY,
13
15
  api_base: process.env.LAS_BASE_URL || 'https://operator.las.cn-beijing.volces.com',
14
16
  dimensions: 2048,
15
17
  encoding_format: 'float'
16
18
  }
17
19
  },
18
- file_storage: { type: 'local', local_base_dir: require('path').join(require('os').homedir(), '.openclaw', 'contextlake', 'files') }
20
+ file_storage: {
21
+ type: 'local',
22
+ local_base_dir: require('path').join(require('os').homedir(), '.openclaw', 'contextlake', 'files'),
23
+ tos: {
24
+ access_key: process.env.VOLCENGINE_ACCESS_KEY || creds.VOLCENGINE_ACCESS_KEY,
25
+ secret_key: process.env.VOLCENGINE_SECRET_KEY || creds.VOLCENGINE_SECRET_KEY
26
+ }
27
+ }
19
28
  };
20
29
  }
@@ -0,0 +1,8 @@
1
+ export interface Credentials {
2
+ LAS_API_KEY?: string;
3
+ VOLCENGINE_ACCESS_KEY?: string;
4
+ VOLCENGINE_SECRET_KEY?: string;
5
+ }
6
+ export declare function loadCredentials(): Credentials;
7
+ export declare function saveCredentials(creds: Credentials): void;
8
+ export declare function promptForInput(promptText: string, defaultVal?: string): Promise<string>;
@@ -0,0 +1,77 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || (function () {
19
+ var ownKeys = function(o) {
20
+ ownKeys = Object.getOwnPropertyNames || function (o) {
21
+ var ar = [];
22
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
+ return ar;
24
+ };
25
+ return ownKeys(o);
26
+ };
27
+ return function (mod) {
28
+ if (mod && mod.__esModule) return mod;
29
+ var result = {};
30
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
+ __setModuleDefault(result, mod);
32
+ return result;
33
+ };
34
+ })();
35
+ Object.defineProperty(exports, "__esModule", { value: true });
36
+ exports.loadCredentials = loadCredentials;
37
+ exports.saveCredentials = saveCredentials;
38
+ exports.promptForInput = promptForInput;
39
+ const fs = __importStar(require("fs"));
40
+ const path = __importStar(require("path"));
41
+ const os = __importStar(require("os"));
42
+ const readline = __importStar(require("readline"));
43
+ const CONFIG_DIR = path.join(os.homedir(), '.openclaw', 'contextlake');
44
+ const CONFIG_FILE = path.join(CONFIG_DIR, 'credentials.json');
45
+ function loadCredentials() {
46
+ if (fs.existsSync(CONFIG_FILE)) {
47
+ try {
48
+ const content = fs.readFileSync(CONFIG_FILE, 'utf-8');
49
+ return JSON.parse(content);
50
+ }
51
+ catch (e) {
52
+ // ignore
53
+ }
54
+ }
55
+ return {};
56
+ }
57
+ function saveCredentials(creds) {
58
+ if (!fs.existsSync(CONFIG_DIR)) {
59
+ fs.mkdirSync(CONFIG_DIR, { recursive: true });
60
+ }
61
+ const current = loadCredentials();
62
+ const updated = { ...current, ...creds };
63
+ fs.writeFileSync(CONFIG_FILE, JSON.stringify(updated, null, 2), { mode: 0o600 });
64
+ }
65
+ async function promptForInput(promptText, defaultVal) {
66
+ const rl = readline.createInterface({
67
+ input: process.stdin,
68
+ output: process.stdout,
69
+ });
70
+ const displayPrompt = defaultVal ? `${promptText} [${defaultVal}]: ` : `${promptText}: `;
71
+ return new Promise((resolve) => {
72
+ rl.question(displayPrompt, (answer) => {
73
+ rl.close();
74
+ resolve(answer.trim() || defaultVal || '');
75
+ });
76
+ });
77
+ }
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "id": "contextlake-openclaw",
3
3
  "name": "ContextLake",
4
- "version": "1.1.0",
4
+ "version": "1.0.3",
5
5
  "description": "A lightweight knowledge base plugin for OpenClaw using LanceDB and TOS, with data profiling support",
6
6
  "skills": ["./src/skills"],
7
7
  "configSchema": {
package/package.json CHANGED
@@ -1,18 +1,14 @@
1
1
  {
2
2
  "name": "@byted-las/contextlake-openclaw",
3
- "version": "1.0.2",
3
+ "version": "1.0.3",
4
4
  "description": "ContextLake OpenClaw Plugin for managing knowledge base",
5
5
  "main": "index.ts",
6
6
  "files": [
7
7
  "dist",
8
- "bin",
9
8
  "index.ts",
10
9
  "src",
11
10
  "openclaw.plugin.json"
12
11
  ],
13
- "bin": {
14
- "contextlake-openclaw": "./bin/contextlake-openclaw.js"
15
- },
16
12
  "openclaw": {
17
13
  "extensions": [
18
14
  "./dist/index.js"
@@ -87,7 +87,7 @@ export class ContextLakeLanceDBClient {
87
87
  return await fallbackQuery.toArray();
88
88
  }
89
89
 
90
- const vector = await this.embeddingProvider.generateEmbedding(query);
90
+ const vector = await this.embeddingProvider.generateMultimodalEmbedding!([{ type: 'text', text: query }]);
91
91
  // @ts-ignore
92
92
  let search = table.vectorSearch(vector).limit(normalizedLimit);
93
93
  if (filter) {
@@ -1,8 +1,9 @@
1
1
  // @ts-ignore
2
- import { ingestAssets } from '../lib/actions/ingest';
2
+ import { ingestSource } from '../lib/actions/ingest';
3
3
  import { retrieveAssets } from '../lib/actions/retrieve';
4
4
  import { listAssets, deleteAssets } from '../lib/actions/manage';
5
5
  import { connectDataSource, ConnectParams } from '../lib/actions/profiler';
6
+ import { loadCredentials, saveCredentials, promptForInput } from '../utils/credentials';
6
7
  import { ContextLakeConfig } from '../utils/config';
7
8
 
8
9
  function parseOptionalInt(value: any, fallback: number): number {
@@ -72,15 +73,11 @@ export function getCliCommands(pluginConfig: ContextLakeConfig, logger: any) {
72
73
  }
73
74
  },
74
75
 
75
- ingestAction: async (files: any, options: any) => {
76
- logger.info(`[${new Date().toISOString()}] [ContextLake] CLI ingest started`, { files, options });
76
+ ingestAction: async (datasource_name: string) => {
77
+ logger.info(`[${new Date().toISOString()}] [ContextLake] CLI ingest started`, { datasource_name });
77
78
  try {
78
- const metadata = parseMetadata(options.metadata);
79
- const result = await ingestAssets({
80
- files,
81
- metadata,
82
- chunkSize: parseOptionalInt(options.chunkSize, 500),
83
- overlap: parseOptionalInt(options.overlap, 50)
79
+ const result = await ingestSource({
80
+ datasource_name
84
81
  }, pluginConfig, logger);
85
82
  // eslint-disable-next-line no-console
86
83
  console.log(JSON.stringify(result, null, 2));
@@ -138,6 +135,35 @@ export function getCliCommands(pluginConfig: ContextLakeConfig, logger: any) {
138
135
  console.error('Error:', e.message);
139
136
  logger.error(`[${new Date().toISOString()}] [ContextLake] CLI delete failed`, { error: e.message, stack: e.stack });
140
137
  }
138
+ },
139
+
140
+ onboardAction: async () => {
141
+ logger.info(`[${new Date().toISOString()}] [ContextLake] CLI onboard started`);
142
+ try {
143
+ const currentCreds = loadCredentials();
144
+ // eslint-disable-next-line no-console
145
+ console.log('Welcome to ContextLake Onboarding!');
146
+ // eslint-disable-next-line no-console
147
+ console.log('Please provide your credentials below. Press enter to keep the current value.');
148
+
149
+ const lasApiKey = await promptForInput('LAS_API_KEY', currentCreds.LAS_API_KEY);
150
+ const volcengineAccessKey = await promptForInput('VOLCENGINE_ACCESS_KEY', currentCreds.VOLCENGINE_ACCESS_KEY);
151
+ const volcengineSecretKey = await promptForInput('VOLCENGINE_SECRET_KEY', currentCreds.VOLCENGINE_SECRET_KEY);
152
+
153
+ const newCreds = {
154
+ LAS_API_KEY: lasApiKey,
155
+ VOLCENGINE_ACCESS_KEY: volcengineAccessKey,
156
+ VOLCENGINE_SECRET_KEY: volcengineSecretKey
157
+ };
158
+
159
+ saveCredentials(newCreds);
160
+ // eslint-disable-next-line no-console
161
+ console.log('Credentials saved successfully!');
162
+ logger.info(`[${new Date().toISOString()}] [ContextLake] CLI onboard success`);
163
+ } catch (e: any) {
164
+ console.error('Error during onboarding:', e.message);
165
+ logger.error(`[${new Date().toISOString()}] [ContextLake] CLI onboard failed`, { error: e.message, stack: e.stack });
166
+ }
141
167
  }
142
168
  };
143
169
  }
@@ -27,6 +27,14 @@ export function registerAll(ctx: OpenClawPluginApi, logger: PluginLogger) {
27
27
  ctx.registerTool(tools.lasDataProfilerTool );
28
28
  logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${tools.lasDataProfilerTool.name}`);
29
29
 
30
+ ctx.registerTool(tools.listDatasourceTool );
31
+ logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${tools.listDatasourceTool.name}`);
32
+
33
+ for (const lasTool of tools.lasTools) {
34
+ ctx.registerTool(lasTool);
35
+ logger.info(`[${new Date().toISOString()}] [ContextLake] Tool registered: ${lasTool.name}`);
36
+ }
37
+
30
38
  } catch (error: any) {
31
39
  logger.error(`[${new Date().toISOString()}] [ContextLake] Error registering agent tools: ${error.message}${error.stack ? '\\n' + error.stack : ''}`);
32
40
  throw error;
@@ -56,11 +64,8 @@ export function registerAll(ctx: OpenClawPluginApi, logger: PluginLogger) {
56
64
  .action(commands.connectAction);
57
65
 
58
66
  // Ingest
59
- contextlake.command('ingest <files...>')
60
- .description('Ingest one or more files into the knowledge base')
61
- .option('-c, --chunk-size <number>', 'Chunk size for text splitting', '500')
62
- .option('-o, --overlap <number>', 'Chunk overlap size', '50')
63
- .option('-m, --metadata <json>', 'JSON metadata to attach to the documents')
67
+ contextlake.command('ingest <datasource_name>')
68
+ .description('Process and ingest all files from a connected data source into the knowledge base')
64
69
  .action(commands.ingestAction);
65
70
 
66
71
  // Search
@@ -83,6 +88,11 @@ export function registerAll(ctx: OpenClawPluginApi, logger: PluginLogger) {
83
88
  .option('--ids <ids...>', 'List of specific file IDs to delete')
84
89
  .option('-f, --filter <string>', 'Filter string to match documents for deletion')
85
90
  .action(commands.deleteAction);
91
+
92
+ // Onboard
93
+ contextlake.command('onboard')
94
+ .description('Configure credentials for ContextLake')
95
+ .action(commands.onboardAction);
86
96
 
87
97
  }, { commands: ['contextlake'] });
88
98
  logger.info(`[${new Date().toISOString()}] [ContextLake] CLI commands registered`);
@@ -102,7 +112,7 @@ export function registerAll(ctx: OpenClawPluginApi, logger: PluginLogger) {
102
112
 
103
113
  ctx.registerCommand({
104
114
  name: 'contextlake-ingest',
105
- description: 'Ingest files into the knowledge base (usage: /contextlake-ingest file1 file2)',
115
+ description: 'Process and ingest all files from a connected data source (usage: /contextlake-ingest <datasource_name>)',
106
116
  acceptsArgs: true,
107
117
  handler: slashCommands.ingestHandler
108
118
  });
@@ -127,6 +137,20 @@ export function registerAll(ctx: OpenClawPluginApi, logger: PluginLogger) {
127
137
  acceptsArgs: true,
128
138
  handler: slashCommands.deleteHandler
129
139
  });
140
+
141
+ ctx.registerCommand({
142
+ name: 'contextlake-profiler',
143
+ description: 'Connect to a data source and profile its structure (usage: /contextlake-profiler <datasource_name> <vendor> <bucket> <prefix>)',
144
+ acceptsArgs: true,
145
+ handler: slashCommands.profilerHandler
146
+ });
147
+
148
+ ctx.registerCommand({
149
+ name: 'contextlake-list-datasource',
150
+ description: 'List all connected and profiled data sources (usage: /contextlake-list-datasource)',
151
+ acceptsArgs: false,
152
+ handler: slashCommands.listDatasourceHandler
153
+ });
130
154
 
131
155
  logger.info(`[${new Date().toISOString()}] [ContextLake] Slash commands registered`);
132
156
  } catch (error: any) {