gigaclaw 1.6.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config/brand.json +23 -0
- package/lib/brand.js +88 -0
- package/lib/chat/components/app-sidebar.js +2 -1
- package/lib/chat/components/app-sidebar.jsx +2 -1
- package/lib/connectors/base.js +175 -0
- package/lib/connectors/filesystem.js +212 -0
- package/lib/connectors/index.js +15 -0
- package/lib/connectors/registry.js +83 -0
- package/lib/rag/chunker.js +151 -0
- package/lib/rag/embeddings.js +201 -0
- package/lib/rag/extractors.js +210 -0
- package/lib/rag/hybrid-search.js +191 -0
- package/lib/rag/index.js +149 -0
- package/lib/rag/vector-store.js +235 -0
- package/lib/rag/watcher.js +238 -0
- package/package.json +37 -1
- package/templates/app/components/ascii-logo.jsx +6 -4
- package/templates/app/layout.js +11 -8
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "GigaClaw",
|
|
3
|
+
"shortName": "GigaClaw",
|
|
4
|
+
"nameLower": "gigaclaw",
|
|
5
|
+
"tagline": "India's Autonomous AI Agent",
|
|
6
|
+
"taglineFull": "India's Autonomous AI Agent · Powered by Gignaati",
|
|
7
|
+
"description": "GigaClaw is an autonomous AI agent platform by Gignaati. Build, deploy, and run AI agents 24/7 with India-first, edge-native AI. Supports PragatiGPT, Claude, GPT, Gemini, and local models via Ollama.",
|
|
8
|
+
"company": "Gignaati",
|
|
9
|
+
"companyUrl": "https://www.gignaati.com",
|
|
10
|
+
"supportEmail": "support@gignaati.com",
|
|
11
|
+
"packageName": "gigaclaw",
|
|
12
|
+
"npmInstallCmd": "npx gigaclaw@latest",
|
|
13
|
+
"localPort": 3000,
|
|
14
|
+
"keywords": ["AI agent", "autonomous agent", "Gignaati", "PragatiGPT", "India AI", "edge AI", "GigaClaw"],
|
|
15
|
+
"social": {
|
|
16
|
+
"github": "https://github.com/gignaati/gigaclaw",
|
|
17
|
+
"website": "https://gigaclaw.gignaati.com"
|
|
18
|
+
},
|
|
19
|
+
"pragatigpt": {
|
|
20
|
+
"label": "PragatiGPT (Gignaati — India-first)",
|
|
21
|
+
"description": "PragatiGPT — India-first, edge-native AI by Gignaati"
|
|
22
|
+
}
|
|
23
|
+
}
|
package/lib/brand.js
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* lib/brand.js — Brand Abstraction Layer
|
|
3
|
+
*
|
|
4
|
+
* Single source of truth for all brand strings in GigaClaw.
|
|
5
|
+
* Reads from config/brand.json at the project root (process.cwd()).
|
|
6
|
+
* Falls back to the package-bundled config/brand.json if the user's
|
|
7
|
+
* project does not have a custom brand.json.
|
|
8
|
+
*
|
|
9
|
+
* Architecture principle: BA-ARCH (Brand-Agnostic Architecture)
|
|
10
|
+
* Rebranding requires only editing config/brand.json — zero source code changes.
|
|
11
|
+
*
|
|
12
|
+
* Usage:
|
|
13
|
+
* import { brand } from '../lib/brand.js';
|
|
14
|
+
* console.log(brand.name); // "GigaClaw"
|
|
15
|
+
* console.log(brand.tagline); // "India's Autonomous AI Agent"
|
|
16
|
+
* console.log(brand.company); // "Gignaati"
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import fs from 'fs';
|
|
20
|
+
import path from 'path';
|
|
21
|
+
import { fileURLToPath } from 'url';
|
|
22
|
+
|
|
23
|
+
const __filename = fileURLToPath(import.meta.url);
|
|
24
|
+
const __dirname = path.dirname(__filename);
|
|
25
|
+
|
|
26
|
+
// Package-bundled brand.json (authoritative default)
|
|
27
|
+
const PACKAGE_BRAND_PATH = path.join(__dirname, '..', 'config', 'brand.json');
|
|
28
|
+
|
|
29
|
+
// User project brand.json (optional override — allows white-labelling)
|
|
30
|
+
const PROJECT_BRAND_PATH = path.join(process.cwd(), 'config', 'brand.json');
|
|
31
|
+
|
|
32
|
+
function loadBrand() {
|
|
33
|
+
// Try user project override first (white-label support)
|
|
34
|
+
if (fs.existsSync(PROJECT_BRAND_PATH)) {
|
|
35
|
+
try {
|
|
36
|
+
const raw = fs.readFileSync(PROJECT_BRAND_PATH, 'utf8');
|
|
37
|
+
return JSON.parse(raw);
|
|
38
|
+
} catch {
|
|
39
|
+
// Fall through to package default
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// Fall back to package-bundled brand.json
|
|
44
|
+
if (fs.existsSync(PACKAGE_BRAND_PATH)) {
|
|
45
|
+
try {
|
|
46
|
+
const raw = fs.readFileSync(PACKAGE_BRAND_PATH, 'utf8');
|
|
47
|
+
return JSON.parse(raw);
|
|
48
|
+
} catch {
|
|
49
|
+
// Fall through to hardcoded defaults
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// Last-resort hardcoded defaults (should never be reached in normal operation)
|
|
54
|
+
return {
|
|
55
|
+
name: 'GigaClaw',
|
|
56
|
+
nameLower: 'gigaclaw',
|
|
57
|
+
tagline: "India's Autonomous AI Agent",
|
|
58
|
+
taglineFull: "India's Autonomous AI Agent · Powered by Gignaati",
|
|
59
|
+
description: 'GigaClaw is an autonomous AI agent platform by Gignaati.',
|
|
60
|
+
company: 'Gignaati',
|
|
61
|
+
companyUrl: 'https://www.gignaati.com',
|
|
62
|
+
packageName: 'gigaclaw',
|
|
63
|
+
npmInstallCmd: 'npx gigaclaw@latest',
|
|
64
|
+
localPort: 3000,
|
|
65
|
+
keywords: ['AI agent', 'autonomous agent', 'GigaClaw'],
|
|
66
|
+
social: {
|
|
67
|
+
github: 'https://github.com/gignaati/gigaclaw',
|
|
68
|
+
website: 'https://gigaclaw.gignaati.com',
|
|
69
|
+
},
|
|
70
|
+
pragatigpt: {
|
|
71
|
+
label: 'PragatiGPT (Gignaati — India-first)',
|
|
72
|
+
description: 'PragatiGPT — India-first, edge-native AI by Gignaati',
|
|
73
|
+
},
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export const brand = loadBrand();
|
|
78
|
+
|
|
79
|
+
// Named convenience exports for the most commonly used fields
|
|
80
|
+
export const BRAND_NAME = brand.name;
|
|
81
|
+
export const BRAND_TAGLINE = brand.tagline;
|
|
82
|
+
export const BRAND_TAGLINE_FULL = brand.taglineFull;
|
|
83
|
+
export const BRAND_COMPANY = brand.company;
|
|
84
|
+
export const BRAND_COMPANY_URL = brand.companyUrl;
|
|
85
|
+
export const BRAND_PACKAGE = brand.packageName;
|
|
86
|
+
export const BRAND_DESCRIPTION = brand.description;
|
|
87
|
+
|
|
88
|
+
export default brand;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use client";
|
|
2
2
|
import { Fragment, jsx, jsxs } from "react/jsx-runtime";
|
|
3
3
|
import { useState, useEffect } from "react";
|
|
4
|
+
import { BRAND_NAME } from "../../brand.js";
|
|
4
5
|
import { CirclePlusIcon, PanelLeftIcon, MessageIcon, BellIcon, SwarmIcon, ArrowUpCircleIcon, LifeBuoyIcon, GitPullRequestIcon, ShieldIcon } from "./icons.js";
|
|
5
6
|
import { getUnreadNotificationCount, getPullRequestCount, getAppVersion } from "../actions.js";
|
|
6
7
|
import { SidebarHistory } from "./sidebar-history.js";
|
|
@@ -52,7 +53,7 @@ function AppSidebar({ user }) {
|
|
|
52
53
|
/* @__PURE__ */ jsxs(SidebarHeader, { children: [
|
|
53
54
|
/* @__PURE__ */ jsxs("div", { className: collapsed ? "flex justify-center" : "flex items-center justify-between", children: [
|
|
54
55
|
!collapsed && /* @__PURE__ */ jsxs("span", { className: "px-2 font-semibold text-lg", children: [
|
|
55
|
-
|
|
56
|
+
BRAND_NAME,
|
|
56
57
|
version && /* @__PURE__ */ jsxs("span", { className: "text-[11px] font-normal text-muted-foreground", children: [
|
|
57
58
|
" v",
|
|
58
59
|
version
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
'use client';
|
|
2
2
|
|
|
3
3
|
import { useState, useEffect } from 'react';
|
|
4
|
+
import { BRAND_NAME } from '../../brand.js';
|
|
4
5
|
import { CirclePlusIcon, PanelLeftIcon, MessageIcon, BellIcon, SwarmIcon, ArrowUpCircleIcon, LifeBuoyIcon, GitPullRequestIcon, ShieldIcon } from './icons.js';
|
|
5
6
|
import { getUnreadNotificationCount, getPullRequestCount, getAppVersion } from '../actions.js';
|
|
6
7
|
import { SidebarHistory } from './sidebar-history.js';
|
|
@@ -63,7 +64,7 @@ export function AppSidebar({ user }) {
|
|
|
63
64
|
{/* Top row: brand name + toggle icon (open) or just toggle icon (collapsed) */}
|
|
64
65
|
<div className={collapsed ? 'flex justify-center' : 'flex items-center justify-between'}>
|
|
65
66
|
{!collapsed && (
|
|
66
|
-
<span className="px-2 font-semibold text-lg">
|
|
67
|
+
<span className="px-2 font-semibold text-lg">{BRAND_NAME}{version && <span className="text-[11px] font-normal text-muted-foreground"> v{version}</span>}</span>
|
|
67
68
|
)}
|
|
68
69
|
<Tooltip>
|
|
69
70
|
<TooltipTrigger asChild>
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* lib/connectors/base.js — BaseConnector Interface
|
|
3
|
+
*
|
|
4
|
+
* All GigaClaw connectors extend this base class.
|
|
5
|
+
* A connector bridges an external data source (filesystem, Notion, GitHub,
|
|
6
|
+
* Confluence, etc.) to the RAG ingestion pipeline.
|
|
7
|
+
*
|
|
8
|
+
* Connector lifecycle:
|
|
9
|
+
* 1. connect() — establish connection, validate credentials
|
|
10
|
+
* 2. listFiles() — enumerate available documents
|
|
11
|
+
* 3. fetchFile() — retrieve a single document's content
|
|
12
|
+
* 4. sync() — full sync: list + fetch + ingest all documents
|
|
13
|
+
* 5. disconnect() — clean up resources
|
|
14
|
+
*
|
|
15
|
+
* Implementing a new connector:
|
|
16
|
+
* 1. Extend BaseConnector
|
|
17
|
+
* 2. Override all abstract methods
|
|
18
|
+
* 3. Register in lib/connectors/registry.js
|
|
19
|
+
* 4. Add a setup step in setup/setup.mjs
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
export class BaseConnector {
|
|
23
|
+
/**
|
|
24
|
+
* @param {Object} config - Connector-specific configuration
|
|
25
|
+
* @param {string} config.id - Unique connector instance ID
|
|
26
|
+
* @param {string} config.name - Human-readable connector name
|
|
27
|
+
*/
|
|
28
|
+
constructor(config = {}) {
|
|
29
|
+
this.id = config.id || this.constructor.name.toLowerCase();
|
|
30
|
+
this.name = config.name || this.constructor.name;
|
|
31
|
+
this.config = config;
|
|
32
|
+
this._connected = false;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Connector type identifier. Override in subclasses.
|
|
37
|
+
* @returns {string}
|
|
38
|
+
*/
|
|
39
|
+
static get type() {
|
|
40
|
+
return 'base';
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Human-readable display name for the connector.
|
|
45
|
+
* @returns {string}
|
|
46
|
+
*/
|
|
47
|
+
static get displayName() {
|
|
48
|
+
return 'Base Connector';
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Configuration schema for the setup wizard.
|
|
53
|
+
* Return an array of field descriptors.
|
|
54
|
+
* @returns {Array<{key: string, label: string, type: 'text' | 'password' | 'path' | 'boolean', required: boolean, default?: any}>}
|
|
55
|
+
*/
|
|
56
|
+
static get configSchema() {
|
|
57
|
+
return [];
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Establish connection to the data source.
|
|
62
|
+
* Validate credentials and test connectivity.
|
|
63
|
+
* @returns {Promise<void>}
|
|
64
|
+
* @throws {Error} If connection fails
|
|
65
|
+
*/
|
|
66
|
+
async connect() {
|
|
67
|
+
throw new Error(`${this.constructor.name}.connect() not implemented`);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* List all documents available from this connector.
|
|
72
|
+
* @param {Object} [options]
|
|
73
|
+
* @param {string} [options.path] - Filter to a specific path/folder
|
|
74
|
+
* @param {string[]} [options.extensions] - Filter by file extension
|
|
75
|
+
* @returns {Promise<Array<{id: string, name: string, path: string, size: number, modifiedAt: Date, mimeType?: string}>>}
|
|
76
|
+
*/
|
|
77
|
+
async listFiles(options = {}) {
|
|
78
|
+
throw new Error(`${this.constructor.name}.listFiles() not implemented`);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Fetch the content of a single document.
|
|
83
|
+
* @param {string} fileId - File identifier from listFiles()
|
|
84
|
+
* @returns {Promise<{text: string, metadata: Object}>}
|
|
85
|
+
*/
|
|
86
|
+
async fetchFile(fileId) {
|
|
87
|
+
throw new Error(`${this.constructor.name}.fetchFile() not implemented`);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Perform a full sync: list all files, fetch content, ingest into RAG.
|
|
92
|
+
* Default implementation calls listFiles() + fetchFile() for each file.
|
|
93
|
+
* Override for connectors with bulk export APIs.
|
|
94
|
+
*
|
|
95
|
+
* @param {Object} [options]
|
|
96
|
+
* @param {boolean} [options.skipIndexed=true] - Skip files already in vector store
|
|
97
|
+
* @param {Function} [options.onProgress] - Callback(current, total, fileName)
|
|
98
|
+
* @returns {Promise<{synced: number, skipped: number, errors: number}>}
|
|
99
|
+
*/
|
|
100
|
+
async sync(options = {}) {
|
|
101
|
+
const { skipIndexed = true, onProgress } = options;
|
|
102
|
+
|
|
103
|
+
if (!this._connected) await this.connect();
|
|
104
|
+
|
|
105
|
+
const files = await this.listFiles();
|
|
106
|
+
const stats = { synced: 0, skipped: 0, errors: 0 };
|
|
107
|
+
|
|
108
|
+
for (let i = 0; i < files.length; i++) {
|
|
109
|
+
const file = files[i];
|
|
110
|
+
if (onProgress) onProgress(i + 1, files.length, file.name);
|
|
111
|
+
|
|
112
|
+
try {
|
|
113
|
+
const { text, metadata } = await this.fetchFile(file.id);
|
|
114
|
+
if (!text || text.trim().length === 0) {
|
|
115
|
+
stats.skipped++;
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// Lazy import to avoid circular dependency
|
|
120
|
+
const { ingest } = await import('../rag/index.js');
|
|
121
|
+
// Write to a temp path for ingestion
|
|
122
|
+
const { default: os } = await import('os');
|
|
123
|
+
const { default: path } = await import('path');
|
|
124
|
+
const { default: fs } = await import('fs');
|
|
125
|
+
const tmpPath = path.join(os.tmpdir(), `gigaclaw-connector-${Date.now()}-${file.name}`);
|
|
126
|
+
fs.writeFileSync(tmpPath, text, 'utf8');
|
|
127
|
+
|
|
128
|
+
try {
|
|
129
|
+
await ingest(tmpPath);
|
|
130
|
+
stats.synced++;
|
|
131
|
+
} finally {
|
|
132
|
+
fs.unlinkSync(tmpPath);
|
|
133
|
+
}
|
|
134
|
+
} catch (err) {
|
|
135
|
+
console.error(`[Connector:${this.id}] Error syncing ${file.name}: ${err.message}`);
|
|
136
|
+
stats.errors++;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
return stats;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Clean up resources (close connections, clear caches).
|
|
145
|
+
* @returns {Promise<void>}
|
|
146
|
+
*/
|
|
147
|
+
async disconnect() {
|
|
148
|
+
this._connected = false;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* Check if the connector is currently connected.
|
|
153
|
+
* @returns {boolean}
|
|
154
|
+
*/
|
|
155
|
+
get isConnected() {
|
|
156
|
+
return this._connected;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Validate the connector configuration.
|
|
161
|
+
* @returns {{valid: boolean, errors: string[]}}
|
|
162
|
+
*/
|
|
163
|
+
validateConfig() {
|
|
164
|
+
const schema = this.constructor.configSchema;
|
|
165
|
+
const errors = [];
|
|
166
|
+
|
|
167
|
+
for (const field of schema) {
|
|
168
|
+
if (field.required && !this.config[field.key]) {
|
|
169
|
+
errors.push(`Missing required field: ${field.key}`);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
return { valid: errors.length === 0, errors };
|
|
174
|
+
}
|
|
175
|
+
}
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* lib/connectors/filesystem.js — Local File System Connector
|
|
3
|
+
*
|
|
4
|
+
* Connects to a local directory and ingests all supported documents
|
|
5
|
+
* into the RAG knowledge base.
|
|
6
|
+
*
|
|
7
|
+
* This is the default connector — it watches ~/gigaclaw-docs/ by default
|
|
8
|
+
* and can be configured to watch any local directory.
|
|
9
|
+
*
|
|
10
|
+
* Configuration:
|
|
11
|
+
* path — Absolute path to the directory to watch (required)
|
|
12
|
+
* watch — Whether to watch for file changes (default: true)
|
|
13
|
+
* recursive — Whether to recurse into subdirectories (default: true)
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import fs from 'fs';
|
|
17
|
+
import path from 'path';
|
|
18
|
+
import os from 'os';
|
|
19
|
+
import { BaseConnector } from './base.js';
|
|
20
|
+
import { isSupportedFile, extractFile, SUPPORTED_EXTENSIONS } from '../rag/extractors.js';
|
|
21
|
+
import { isSourceIndexed } from '../rag/vector-store.js';
|
|
22
|
+
import { startWatcher, stopWatcher, ingestFile } from '../rag/watcher.js';
|
|
23
|
+
|
|
24
|
+
const DEFAULT_DOCS_DIR = path.join(os.homedir(), 'gigaclaw-docs');
|
|
25
|
+
|
|
26
|
+
export class FilesystemConnector extends BaseConnector {
|
|
27
|
+
/**
|
|
28
|
+
* @param {Object} config
|
|
29
|
+
* @param {string} [config.path] - Directory to connect to
|
|
30
|
+
* @param {boolean} [config.watch=true] - Watch for changes
|
|
31
|
+
* @param {boolean} [config.recursive=true] - Recurse into subdirectories
|
|
32
|
+
*/
|
|
33
|
+
constructor(config = {}) {
|
|
34
|
+
super({
|
|
35
|
+
id: 'filesystem',
|
|
36
|
+
name: 'Local File System',
|
|
37
|
+
...config,
|
|
38
|
+
});
|
|
39
|
+
this.docsPath = config.path || process.env.RAG_DOCS_DIR || DEFAULT_DOCS_DIR;
|
|
40
|
+
this.watch = config.watch !== false;
|
|
41
|
+
this.recursive = config.recursive !== false;
|
|
42
|
+
this._watcherStarted = false;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
static get type() {
|
|
46
|
+
return 'filesystem';
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
static get displayName() {
|
|
50
|
+
return 'Local File System';
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
static get configSchema() {
|
|
54
|
+
return [
|
|
55
|
+
{
|
|
56
|
+
key: 'path',
|
|
57
|
+
label: 'Documents directory',
|
|
58
|
+
type: 'path',
|
|
59
|
+
required: false,
|
|
60
|
+
default: DEFAULT_DOCS_DIR,
|
|
61
|
+
description: 'Directory containing documents to index. Defaults to ~/gigaclaw-docs/',
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
key: 'watch',
|
|
65
|
+
label: 'Watch for changes',
|
|
66
|
+
type: 'boolean',
|
|
67
|
+
required: false,
|
|
68
|
+
default: true,
|
|
69
|
+
description: 'Automatically re-index files when they change',
|
|
70
|
+
},
|
|
71
|
+
{
|
|
72
|
+
key: 'recursive',
|
|
73
|
+
label: 'Include subdirectories',
|
|
74
|
+
type: 'boolean',
|
|
75
|
+
required: false,
|
|
76
|
+
default: true,
|
|
77
|
+
description: 'Recurse into subdirectories',
|
|
78
|
+
},
|
|
79
|
+
];
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
async connect() {
|
|
83
|
+
// Ensure the directory exists
|
|
84
|
+
if (!fs.existsSync(this.docsPath)) {
|
|
85
|
+
fs.mkdirSync(this.docsPath, { recursive: true });
|
|
86
|
+
console.log(`[FilesystemConnector] Created docs directory: ${this.docsPath}`);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
this._connected = true;
|
|
90
|
+
console.log(`[FilesystemConnector] Connected to: ${this.docsPath}`);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* List all supported files in the configured directory.
|
|
95
|
+
* @param {Object} [options]
|
|
96
|
+
* @param {string[]} [options.extensions] - Filter by extension
|
|
97
|
+
* @returns {Promise<Array<{id: string, name: string, path: string, size: number, modifiedAt: Date}>>}
|
|
98
|
+
*/
|
|
99
|
+
async listFiles(options = {}) {
|
|
100
|
+
if (!this._connected) await this.connect();
|
|
101
|
+
|
|
102
|
+
const { extensions } = options;
|
|
103
|
+
const allowedExts = extensions
|
|
104
|
+
? new Set(extensions.map(e => e.startsWith('.') ? e : `.${e}`))
|
|
105
|
+
: SUPPORTED_EXTENSIONS;
|
|
106
|
+
|
|
107
|
+
const files = [];
|
|
108
|
+
this._walkDir(this.docsPath, files, allowedExts);
|
|
109
|
+
return files;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Recursively walk the directory and collect file metadata.
|
|
114
|
+
* @private
|
|
115
|
+
*/
|
|
116
|
+
_walkDir(dirPath, files, allowedExts) {
|
|
117
|
+
try {
|
|
118
|
+
const entries = fs.readdirSync(dirPath, { withFileTypes: true });
|
|
119
|
+
for (const entry of entries) {
|
|
120
|
+
if (entry.name.startsWith('.')) continue; // Skip hidden files/dirs
|
|
121
|
+
|
|
122
|
+
const fullPath = path.join(dirPath, entry.name);
|
|
123
|
+
|
|
124
|
+
if (entry.isDirectory() && this.recursive) {
|
|
125
|
+
this._walkDir(fullPath, files, allowedExts);
|
|
126
|
+
} else if (entry.isFile()) {
|
|
127
|
+
const ext = path.extname(entry.name).toLowerCase();
|
|
128
|
+
if (allowedExts.has(ext)) {
|
|
129
|
+
const stat = fs.statSync(fullPath);
|
|
130
|
+
files.push({
|
|
131
|
+
id: fullPath,
|
|
132
|
+
name: entry.name,
|
|
133
|
+
path: fullPath,
|
|
134
|
+
size: stat.size,
|
|
135
|
+
modifiedAt: stat.mtime,
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
} catch {
|
|
141
|
+
// Skip unreadable directories
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Fetch the content of a file by its path (used as the file ID).
|
|
147
|
+
* @param {string} fileId - Absolute file path
|
|
148
|
+
* @returns {Promise<{text: string, metadata: Object}>}
|
|
149
|
+
*/
|
|
150
|
+
async fetchFile(fileId) {
|
|
151
|
+
return extractFile(fileId);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Sync all files in the directory into the RAG knowledge base.
|
|
156
|
+
* @param {Object} [options]
|
|
157
|
+
* @param {boolean} [options.skipIndexed=true]
|
|
158
|
+
* @param {Function} [options.onProgress]
|
|
159
|
+
* @returns {Promise<{synced: number, skipped: number, errors: number}>}
|
|
160
|
+
*/
|
|
161
|
+
async sync(options = {}) {
|
|
162
|
+
const { skipIndexed = true, onProgress } = options;
|
|
163
|
+
|
|
164
|
+
if (!this._connected) await this.connect();
|
|
165
|
+
|
|
166
|
+
const files = await this.listFiles();
|
|
167
|
+
const stats = { synced: 0, skipped: 0, errors: 0 };
|
|
168
|
+
|
|
169
|
+
for (let i = 0; i < files.length; i++) {
|
|
170
|
+
const file = files[i];
|
|
171
|
+
if (onProgress) onProgress(i + 1, files.length, file.name);
|
|
172
|
+
|
|
173
|
+
if (skipIndexed && isSourceIndexed(file.path)) {
|
|
174
|
+
stats.skipped++;
|
|
175
|
+
continue;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
const result = await ingestFile(file.path);
|
|
179
|
+
if (result.error) {
|
|
180
|
+
stats.errors++;
|
|
181
|
+
} else if (result.skipped) {
|
|
182
|
+
stats.skipped++;
|
|
183
|
+
} else {
|
|
184
|
+
stats.synced++;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// Start watcher if configured
|
|
189
|
+
if (this.watch && !this._watcherStarted) {
|
|
190
|
+
await startWatcher(this.docsPath);
|
|
191
|
+
this._watcherStarted = true;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
return stats;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
async disconnect() {
|
|
198
|
+
if (this._watcherStarted) {
|
|
199
|
+
stopWatcher();
|
|
200
|
+
this._watcherStarted = false;
|
|
201
|
+
}
|
|
202
|
+
await super.disconnect();
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Get the docs directory path.
|
|
207
|
+
* @returns {string}
|
|
208
|
+
*/
|
|
209
|
+
get docsDirectory() {
|
|
210
|
+
return this.docsPath;
|
|
211
|
+
}
|
|
212
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* lib/connectors/index.js — Connector Framework Public API
|
|
3
|
+
*
|
|
4
|
+
* Export all connector classes and registry functions.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
export { BaseConnector } from './base.js';
|
|
8
|
+
export { FilesystemConnector } from './filesystem.js';
|
|
9
|
+
export {
|
|
10
|
+
registerConnector,
|
|
11
|
+
getConnectorClass,
|
|
12
|
+
listConnectors,
|
|
13
|
+
createConnector,
|
|
14
|
+
createDefaultConnector,
|
|
15
|
+
} from './registry.js';
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* lib/connectors/registry.js — Connector Registry
|
|
3
|
+
*
|
|
4
|
+
* Central registry for all available connectors.
|
|
5
|
+
* Connectors are registered here and can be instantiated by type.
|
|
6
|
+
*
|
|
7
|
+
* Built-in connectors:
|
|
8
|
+
* filesystem — Local File System (default)
|
|
9
|
+
*
|
|
10
|
+
* Future connectors (v1.8+):
|
|
11
|
+
* notion — Notion workspace
|
|
12
|
+
* github — GitHub repositories
|
|
13
|
+
* confluence — Atlassian Confluence
|
|
14
|
+
* slack — Slack channels
|
|
15
|
+
* gdrive — Google Drive
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import { FilesystemConnector } from './filesystem.js';
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Registry of all available connector classes.
|
|
22
|
+
* Key: connector type string
|
|
23
|
+
* Value: connector class
|
|
24
|
+
*/
|
|
25
|
+
const REGISTRY = new Map([
|
|
26
|
+
['filesystem', FilesystemConnector],
|
|
27
|
+
]);
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Register a custom connector class.
|
|
31
|
+
* @param {string} type - Unique type identifier
|
|
32
|
+
* @param {typeof import('./base.js').BaseConnector} ConnectorClass
|
|
33
|
+
*/
|
|
34
|
+
export function registerConnector(type, ConnectorClass) {
|
|
35
|
+
REGISTRY.set(type, ConnectorClass);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Get a connector class by type.
|
|
40
|
+
* @param {string} type
|
|
41
|
+
* @returns {typeof import('./base.js').BaseConnector | undefined}
|
|
42
|
+
*/
|
|
43
|
+
export function getConnectorClass(type) {
|
|
44
|
+
return REGISTRY.get(type);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* List all registered connector types with their display names.
|
|
49
|
+
* @returns {Array<{type: string, displayName: string, configSchema: Array}>}
|
|
50
|
+
*/
|
|
51
|
+
export function listConnectors() {
|
|
52
|
+
return Array.from(REGISTRY.entries()).map(([type, cls]) => ({
|
|
53
|
+
type,
|
|
54
|
+
displayName: cls.displayName,
|
|
55
|
+
configSchema: cls.configSchema,
|
|
56
|
+
}));
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Create a connector instance by type.
|
|
61
|
+
* @param {string} type - Connector type
|
|
62
|
+
* @param {Object} config - Connector configuration
|
|
63
|
+
* @returns {import('./base.js').BaseConnector}
|
|
64
|
+
* @throws {Error} If connector type is not registered
|
|
65
|
+
*/
|
|
66
|
+
export function createConnector(type, config = {}) {
|
|
67
|
+
const ConnectorClass = REGISTRY.get(type);
|
|
68
|
+
if (!ConnectorClass) {
|
|
69
|
+
throw new Error(`Unknown connector type: "${type}". Available: ${[...REGISTRY.keys()].join(', ')}`);
|
|
70
|
+
}
|
|
71
|
+
return new ConnectorClass(config);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Create and connect the default filesystem connector.
|
|
76
|
+
* @param {Object} [config]
|
|
77
|
+
* @returns {Promise<import('./filesystem.js').FilesystemConnector>}
|
|
78
|
+
*/
|
|
79
|
+
export async function createDefaultConnector(config = {}) {
|
|
80
|
+
const connector = new FilesystemConnector(config);
|
|
81
|
+
await connector.connect();
|
|
82
|
+
return connector;
|
|
83
|
+
}
|