@thinkbase/sdk 1.0.0 → 1.0.1

package/dist/examples/basic.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/examples/basic.js

@@ -0,0 +1,82 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const dotenv = __importStar(require("dotenv"));
+const path = __importStar(require("path"));
+const src_1 = require("../src");
+dotenv.config({ path: path.join(__dirname, '..', '.env') });
+async function basicExample() {
+    console.log('ThinkBase SDK - Basic Example\n');
+    const apiKey = process.env.OPENAI_API_KEY;
+    if (!apiKey || apiKey === 'your-openai-api-key-here') {
+        console.error('ERROR: OPENAI_API_KEY not found or using placeholder value');
+        console.error('Current working directory:', process.cwd());
+        console.error('Looking for .env at:', path.join(__dirname, '..', '.env'));
+        console.error('Make sure your .env file has: OPENAI_API_KEY=sk-your-actual-key');
+        process.exit(1);
+    }
+    console.log('✓ API key loaded:', apiKey.substring(0, 10) + '...\n');
+    const thinkbase = new src_1.ThinkBase();
+    const dataset = thinkbase.createDataset({
+        source: 'json',
+        data: [
+            { id: 1, title: 'Bitcoin reaches new highs', content: 'Bitcoin price surges past $50k' },
+            { id: 2, title: 'Ethereum updates', content: 'Ethereum 2.0 successfully deployed' },
+            { id: 3, title: 'New DeFi protocol', content: 'Revolutionary lending platform launches' },
+            { id: 4, title: 'NFT marketplace', content: 'New NFT platform sees record trading' },
+            { id: 5, title: 'Crypto regulation', content: 'New guidelines for crypto exchanges' }
+        ]
+    });
+    console.log('Syncing dataset with embeddings...');
+    await dataset.sync({
+        embedding: 'openai:text-embedding-3-small',
+        chunkSize: 500
+    });
+    console.log('Stats:', dataset.getStats());
+    console.log('\nSearching for: "bitcoin price"');
+    const results = await dataset.search('bitcoin price', { limit: 3 });
+    results.forEach((result, i) => {
+        console.log(`\n[${i + 1}] Score: ${result.score.toFixed(3)}`);
+        console.log(`Content: ${result.content}`);
+    });
+    console.log('\nBuilding context for LLM...');
+    const context = await dataset.context({
+        query: 'cryptocurrency news',
+        limit: 3
+    });
+    console.log('\nContext:');
+    console.log(context);
+}
+basicExample().catch(console.error);

package/dist/examples/llm-context.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/examples/llm-context.js

@@ -0,0 +1,107 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const dotenv = __importStar(require("dotenv"));
+const path = __importStar(require("path"));
+const src_1 = require("../src");
+const openai_1 = __importDefault(require("openai"));
+dotenv.config({ path: path.join(__dirname, '..', '.env') });
+async function llmContextExample() {
+    console.log('ThinkBase SDK - LLM Context Example\n');
+    const thinkbase = new src_1.ThinkBase();
+    const openai = new openai_1.default({ apiKey: process.env.OPENAI_API_KEY });
+    const dataset = thinkbase.createDataset({
+        source: 'json',
+        data: [
+            {
+                symbol: 'BTC',
+                name: 'Bitcoin',
+                price: 50000,
+                change24h: 5.2,
+                marketCap: 950000000000,
+                description: 'The first and largest cryptocurrency by market cap'
+            },
+            {
+                symbol: 'ETH',
+                name: 'Ethereum',
+                price: 3000,
+                change24h: 3.8,
+                marketCap: 360000000000,
+                description: 'Leading smart contract platform'
+            },
+            {
+                symbol: 'SOL',
+                name: 'Solana',
+                price: 100,
+                change24h: -2.1,
+                marketCap: 45000000000,
+                description: 'High-performance blockchain for DeFi and NFTs'
+            }
+        ]
+    });
+    console.log('Syncing dataset...');
+    await dataset.sync({
+        embedding: 'openai:text-embedding-3-small',
+        chunkSize: 500
+    });
+    console.log('Building context for LLM...');
+    const context = await dataset.context({
+        query: 'top performing cryptocurrencies',
+        limit: 3,
+        maxTokens: 2000
+    });
+    console.log('\nContext prepared for LLM:');
+    console.log(context);
+    console.log('\nSending to GPT-4...');
+    const response = await openai.chat.completions.create({
+        model: 'gpt-4',
+        messages: [
+            {
+                role: 'system',
+                content: `You are a crypto market analyst. Use this context:\n\n${context}`
+            },
+            {
+                role: 'user',
+                content: 'What are the key insights about these cryptocurrencies?'
+            }
+        ]
+    });
+    console.log('\nAI Response:');
+    console.log(response.choices[0].message.content);
+}
+llmContextExample().catch(console.error);

package/dist/examples/postrgres.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/examples/postrgres.js

@@ -0,0 +1,73 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const dotenv = __importStar(require("dotenv"));
+const path = __importStar(require("path"));
+const src_1 = require("../src");
+dotenv.config({ path: path.join(__dirname, '..', '.env') });
+async function postgresExample() {
+    console.log('ThinkBase SDK - PostgreSQL Example\n');
+    const thinkbase = new src_1.ThinkBase();
+    const dataset = thinkbase.createDataset({
+        source: 'postgres',
+        connection: process.env.DATABASE_URL || 'postgresql://localhost:5432/mydb',
+        query: 'SELECT * FROM products WHERE active = true'
+    });
+    console.log('Connecting to PostgreSQL...');
+    await dataset.connect();
+    console.log('Syncing dataset...');
+    await dataset.sync({
+        embedding: 'openai:text-embedding-3-small',
+        chunkSize: 1000,
+        batchSize: 50
+    });
+    console.log('Dataset stats:', dataset.getStats());
+    console.log('\nSearching for: "best selling products"');
+    const searchResults = await dataset.search('best selling products', {
+        limit: 5,
+        threshold: 0.7
+    });
+    searchResults.forEach((result, i) => {
+        console.log(`\n[${i + 1}] ${result.score.toFixed(3)}`);
+        console.log(result.content);
+    });
+    console.log('\nAsking natural language query...');
+    const queryResult = await dataset.ask('What are the top 5 most expensive products?');
+    console.log('\nGenerated SQL:', queryResult.sql);
+    console.log('Results:', JSON.stringify(queryResult.data, null, 2));
+    await dataset.disconnect();
+    console.log('\nDisconnected from database');
+}
+postgresExample().catch(console.error);

package/dist/src/client.d.ts

@@ -0,0 +1,8 @@
+import { Dataset } from './dataset';
+import { ThinkBaseConfig, DataSourceConfig } from './types';
+export declare class ThinkBase {
+    private config;
+    constructor(config?: ThinkBaseConfig);
+    createDataset(config: DataSourceConfig): Dataset;
+    getConfig(): ThinkBaseConfig;
+}

package/dist/src/client.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ThinkBase = void 0;
+const dataset_1 = require("./dataset");
+class ThinkBase {
+    constructor(config = {}) {
+        this.config = config;
+    }
+    createDataset(config) {
+        return new dataset_1.Dataset(config);
+    }
+    getConfig() {
+        return this.config;
+    }
+}
+exports.ThinkBase = ThinkBase;

package/dist/src/connectors/base.d.ts

@@ -0,0 +1,9 @@
+import { DataSourceConfig } from '../types';
+export declare abstract class DataConnector {
+    protected config: DataSourceConfig;
+    constructor(config: DataSourceConfig);
+    abstract connect(): Promise<void>;
+    abstract disconnect(): Promise<void>;
+    abstract fetchData(): Promise<any[]>;
+    abstract getName(): string;
+}

package/dist/src/connectors/base.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DataConnector = void 0;
+class DataConnector {
+    constructor(config) {
+        this.config = config;
+    }
+}
+exports.DataConnector = DataConnector;

package/dist/src/connectors/factory.d.ts

@@ -0,0 +1,5 @@
+import { DataConnector } from './base';
+import { DataSourceConfig } from '../types';
+export declare class ConnectorFactory {
+    static create(config: DataSourceConfig): DataConnector;
+}

package/dist/src/connectors/factory.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ConnectorFactory = void 0;
+const postgres_1 = require("./postgres");
+const mysql_1 = require("./mysql");
+const mongodb_1 = require("./mongodb");
+const rest_1 = require("./rest");
+const json_1 = require("./json");
+class ConnectorFactory {
+    static create(config) {
+        switch (config.source) {
+            case 'postgres':
+                return new postgres_1.PostgresConnector(config);
+            case 'mysql':
+                return new mysql_1.MySQLConnector(config);
+            case 'mongodb':
+                return new mongodb_1.MongoDBConnector(config);
+            case 'rest':
+            case 'graphql':
+                return new rest_1.RESTConnector(config);
+            case 'json':
+            case 'csv':
+                return new json_1.JSONConnector(config);
+            default:
+                throw new Error(`Unsupported data source: ${config.source}`);
+        }
+    }
+}
+exports.ConnectorFactory = ConnectorFactory;

package/dist/src/connectors/json.d.ts

@@ -0,0 +1,7 @@
+import { DataConnector } from './base';
+export declare class JSONConnector extends DataConnector {
+    connect(): Promise<void>;
+    disconnect(): Promise<void>;
+    fetchData(): Promise<any[]>;
+    getName(): string;
+}

package/dist/src/connectors/json.js

@@ -0,0 +1,22 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.JSONConnector = void 0;
+const base_1 = require("./base");
+class JSONConnector extends base_1.DataConnector {
+    async connect() {
+        // No connection needed for in-memory data
+    }
+    async disconnect() {
+        // No connection to close
+    }
+    async fetchData() {
+        if (!this.config.data) {
+            throw new Error('JSON data is required');
+        }
+        return Array.isArray(this.config.data) ? this.config.data : [this.config.data];
+    }
+    getName() {
+        return 'JSON';
+    }
+}
+exports.JSONConnector = JSONConnector;

package/dist/src/connectors/mongodb.d.ts

@@ -0,0 +1,9 @@
+import { DataConnector } from './base';
+export declare class MongoDBConnector extends DataConnector {
+    private client;
+    private db;
+    connect(): Promise<void>;
+    disconnect(): Promise<void>;
+    fetchData(): Promise<any[]>;
+    getName(): string;
+}

package/dist/src/connectors/mongodb.js

@@ -0,0 +1,39 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MongoDBConnector = void 0;
+const mongodb_1 = require("mongodb");
+const base_1 = require("./base");
+class MongoDBConnector extends base_1.DataConnector {
+    constructor() {
+        super(...arguments);
+        this.client = null;
+        this.db = null;
+    }
+    async connect() {
+        if (!this.config.connection) {
+            throw new Error('MongoDB connection string is required');
+        }
+        this.client = new mongodb_1.MongoClient(this.config.connection);
+        await this.client.connect();
+        this.db = this.client.db();
+    }
+    async disconnect() {
+        if (this.client) {
+            await this.client.close();
+            this.client = null;
+            this.db = null;
+        }
+    }
+    async fetchData() {
+        if (!this.db) {
+            throw new Error('Not connected to MongoDB');
+        }
+        const collection = this.config.collection || 'data';
+        const documents = await this.db.collection(collection).find({}).toArray();
+        return documents;
+    }
+    getName() {
+        return 'MongoDB';
+    }
+}
+exports.MongoDBConnector = MongoDBConnector;

package/dist/src/connectors/mysql.d.ts

@@ -0,0 +1,8 @@
+import { DataConnector } from './base';
+export declare class MySQLConnector extends DataConnector {
+    private connection;
+    connect(): Promise<void>;
+    disconnect(): Promise<void>;
+    fetchData(): Promise<any[]>;
+    getName(): string;
+}

package/dist/src/connectors/mysql.js

@@ -0,0 +1,38 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MySQLConnector = void 0;
+const promise_1 = __importDefault(require("mysql2/promise"));
+const base_1 = require("./base");
+class MySQLConnector extends base_1.DataConnector {
+    constructor() {
+        super(...arguments);
+        this.connection = null;
+    }
+    async connect() {
+        if (!this.config.connection) {
+            throw new Error('MySQL connection string is required');
+        }
+        this.connection = await promise_1.default.createConnection(this.config.connection);
+    }
+    async disconnect() {
+        if (this.connection) {
+            await this.connection.end();
+            this.connection = null;
+        }
+    }
+    async fetchData() {
+        if (!this.connection) {
+            throw new Error('Not connected to MySQL');
+        }
+        const query = this.config.query || `SELECT * FROM ${this.config.table || 'data'}`;
+        const [rows] = await this.connection.execute(query);
+        return rows;
+    }
+    getName() {
+        return 'MySQL';
+    }
+}
+exports.MySQLConnector = MySQLConnector;

package/dist/src/connectors/postgres.d.ts

@@ -0,0 +1,8 @@
+import { DataConnector } from './base';
+export declare class PostgresConnector extends DataConnector {
+    private pool;
+    connect(): Promise<void>;
+    disconnect(): Promise<void>;
+    fetchData(): Promise<any[]>;
+    getName(): string;
+}

package/dist/src/connectors/postgres.js

@@ -0,0 +1,35 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PostgresConnector = void 0;
+const pg_1 = require("pg");
+const base_1 = require("./base");
+class PostgresConnector extends base_1.DataConnector {
+    constructor() {
+        super(...arguments);
+        this.pool = null;
+    }
+    async connect() {
+        if (!this.config.connection) {
+            throw new Error('PostgreSQL connection string is required');
+        }
+        this.pool = new pg_1.Pool({ connectionString: this.config.connection });
+    }
+    async disconnect() {
+        if (this.pool) {
+            await this.pool.end();
+            this.pool = null;
+        }
+    }
+    async fetchData() {
+        if (!this.pool) {
+            throw new Error('Not connected to PostgreSQL');
+        }
+        const query = this.config.query || `SELECT * FROM ${this.config.table || 'data'}`;
+        const result = await this.pool.query(query);
+        return result.rows;
+    }
+    getName() {
+        return 'PostgreSQL';
+    }
+}
+exports.PostgresConnector = PostgresConnector;

package/dist/src/connectors/rest.d.ts

@@ -0,0 +1,7 @@
+import { DataConnector } from './base';
+export declare class RESTConnector extends DataConnector {
+    connect(): Promise<void>;
+    disconnect(): Promise<void>;
+    fetchData(): Promise<any[]>;
+    getName(): string;
+}

package/dist/src/connectors/rest.js

@@ -0,0 +1,27 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RESTConnector = void 0;
+const axios_1 = __importDefault(require("axios"));
+const base_1 = require("./base");
+class RESTConnector extends base_1.DataConnector {
+    async connect() {
+        // No persistent connection needed for REST
+    }
+    async disconnect() {
+        // No persistent connection to close
+    }
+    async fetchData() {
+        if (!this.config.url) {
+            throw new Error('REST API URL is required');
+        }
+        const response = await axios_1.default.get(this.config.url);
+        return Array.isArray(response.data) ? response.data : [response.data];
+    }
+    getName() {
+        return 'REST API';
+    }
+}
+exports.RESTConnector = RESTConnector;

package/dist/src/context/builder.d.ts

@@ -0,0 +1,11 @@
+import { SearchResult, ContextOptions } from '../types';
+import { DataChunk } from '../types';
+export declare class ContextBuilder {
+    private search;
+    private embedder;
+    constructor(chunks: DataChunk[], embeddingModel: string);
+    build(options: ContextOptions): Promise<string>;
+    private formatContext;
+    private estimateTokens;
+    buildStructured(options: ContextOptions): Promise<SearchResult[]>;
+}

package/dist/src/context/builder.js

@@ -0,0 +1,40 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ContextBuilder = void 0;
+const embedder_1 = require("../embedding/embedder");
+const vector_1 = require("../search/vector");
+class ContextBuilder {
+    constructor(chunks, embeddingModel) {
+        this.search = new vector_1.VectorSearch(chunks);
+        this.embedder = new embedder_1.Embedder(embeddingModel);
+    }
+    async build(options) {
+        const queryEmbedding = await this.embedder.embed(options.query);
+        const results = this.search.search(queryEmbedding, options.limit || 5, 0.5);
+        return this.formatContext(results, options);
+    }
+    formatContext(results, options) {
+        const maxTokens = options.maxTokens || 4000;
+        let context = `Relevant context for: "${options.query}"\n\n`;
+        let currentTokens = this.estimateTokens(context);
+        for (let i = 0; i < results.length; i++) {
+            const result = results[i];
+            const chunk = `[${i + 1}] (score: ${result.score.toFixed(3)})\n${result.content}\n\n`;
+            const chunkTokens = this.estimateTokens(chunk);
+            if (currentTokens + chunkTokens > maxTokens) {
+                break;
+            }
+            context += chunk;
+            currentTokens += chunkTokens;
+        }
+        return context.trim();
+    }
+    estimateTokens(text) {
+        return Math.ceil(text.length / 4);
+    }
+    async buildStructured(options) {
+        const queryEmbedding = await this.embedder.embed(options.query);
+        return this.search.search(queryEmbedding, options.limit || 5, 0.5);
+    }
+}
+exports.ContextBuilder = ContextBuilder;

package/dist/src/dataset.d.ts

@@ -0,0 +1,21 @@
+import { DataSourceConfig, EmbeddingConfig, SearchOptions, ContextOptions, DataChunk, SearchResult, QueryResult } from './types';
+export declare class Dataset {
+    private connector;
+    private chunks;
+    private embeddingConfig?;
+    private nlQuery?;
+    private contextBuilder?;
+    constructor(config: DataSourceConfig);
+    connect(): Promise<void>;
+    disconnect(): Promise<void>;
+    sync(config: EmbeddingConfig): Promise<void>;
+    search(query: string, options?: SearchOptions): Promise<SearchResult[]>;
+    ask(query: string): Promise<QueryResult>;
+    context(options: ContextOptions): Promise<string>;
+    contextStructured(options: ContextOptions): Promise<SearchResult[]>;
+    getChunks(): DataChunk[];
+    getStats(): {
+        totalChunks: number;
+        totalDocuments: number;
+    };
+}

package/dist/src/dataset.js

@@ -0,0 +1,83 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Dataset = void 0;
+const factory_1 = require("./connectors/factory");
+const pipeline_1 = require("./embedding/pipeline");
+const vector_1 = require("./search/vector");
+const natural_language_1 = require("./query/natural-language");
+const builder_1 = require("./context/builder");
+const embedder_1 = require("./embedding/embedder");
+class Dataset {
+    constructor(config) {
+        this.chunks = [];
+        this.connector = factory_1.ConnectorFactory.create(config);
+    }
+    async connect() {
+        await this.connector.connect();
+    }
+    async disconnect() {
+        await this.connector.disconnect();
+    }
+    async sync(config) {
+        this.embeddingConfig = config;
+        await this.connector.connect();
+        const data = await this.connector.fetchData();
+        await this.connector.disconnect();
+        const pipeline = new pipeline_1.EmbeddingPipeline(config);
+        this.chunks = await pipeline.process(data);
+        this.contextBuilder = new builder_1.ContextBuilder(this.chunks, config.embedding);
+        this.nlQuery = new natural_language_1.NaturalLanguageQuery();
+    }
+    async search(query, options = {}) {
+        if (!this.embeddingConfig) {
+            throw new Error('Dataset not synced. Call sync() first.');
+        }
+        const embedder = new embedder_1.Embedder(this.embeddingConfig.embedding);
+        const queryEmbedding = await embedder.embed(query);
+        const search = new vector_1.VectorSearch(this.chunks);
+        if (options.hybrid) {
+            const keywords = query.toLowerCase().split(' ');
+            return search.hybridSearch(queryEmbedding, keywords, options.limit || 10, 0.7);
+        }
+        return search.search(queryEmbedding, options.limit || 10, options.threshold || 0.7);
+    }
+    async ask(query) {
+        if (!this.nlQuery) {
+            this.nlQuery = new natural_language_1.NaturalLanguageQuery();
+        }
+        await this.connector.connect();
+        try {
+            const result = await this.nlQuery.executeQuery(query, async (sql) => {
+                return await this.connector.fetchData();
+            });
+            await this.connector.disconnect();
+            return result;
+        }
+        catch (error) {
+            await this.connector.disconnect();
+            throw error;
+        }
+    }
+    async context(options) {
+        if (!this.contextBuilder) {
+            throw new Error('Dataset not synced. Call sync() first.');
+        }
+        return this.contextBuilder.build(options);
+    }
+    async contextStructured(options) {
+        if (!this.contextBuilder) {
+            throw new Error('Dataset not synced. Call sync() first.');
+        }
+        return this.contextBuilder.buildStructured(options);
+    }
+    getChunks() {
+        return this.chunks;
+    }
+    getStats() {
+        return {
+            totalChunks: this.chunks.length,
+            totalDocuments: this.chunks.length
+        };
+    }
+}
+exports.Dataset = Dataset;

package/dist/src/embedding/chunker.d.ts

@@ -0,0 +1,10 @@
+export declare class TextChunker {
+    private chunkSize;
+    private chunkOverlap;
+    constructor(chunkSize?: number, chunkOverlap?: number);
+    chunk(text: string): string[];
+    chunkDocuments(documents: any[]): Array<{
+        content: string;
+        metadata: any;
+    }>;
+}

package/dist/src/embedding/chunker.js

@@ -0,0 +1,41 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TextChunker = void 0;
+class TextChunker {
+    constructor(chunkSize = 1000, chunkOverlap = 200) {
+        this.chunkSize = chunkSize;
+        this.chunkOverlap = chunkOverlap;
+    }
+    chunk(text) {
+        const chunks = [];
+        let start = 0;
+        while (start < text.length) {
+            const end = Math.min(start + this.chunkSize, text.length);
+            let chunk = text.slice(start, end);
+            if (end < text.length) {
+                const lastSpace = chunk.lastIndexOf(' ');
+                if (lastSpace > 0) {
+                    chunk = chunk.slice(0, lastSpace);
+                }
+            }
+            chunks.push(chunk.trim());
+            start += this.chunkSize - this.chunkOverlap;
+        }
+        return chunks.filter(c => c.length > 0);
+    }
+    chunkDocuments(documents) {
+        const chunkedDocs = [];
+        for (const doc of documents) {
+            const text = typeof doc === 'string' ? doc : JSON.stringify(doc);
+            const chunks = this.chunk(text);
+            for (const chunk of chunks) {
+                chunkedDocs.push({
+                    content: chunk,
+                    metadata: typeof doc === 'object' ? doc : {}
+                });
+            }
+        }
+        return chunkedDocs;
+    }
+}
+exports.TextChunker = TextChunker;

package/dist/src/embedding/embedder.d.ts

@@ -0,0 +1,9 @@
+export declare class Embedder {
+    private model;
+    private client;
+    constructor(model: string);
+    private isOpenAIModel;
+    private getModelName;
+    embed(text: string): Promise<number[]>;
+    embedBatch(texts: string[], batchSize?: number): Promise<number[][]>;
+}

package/dist/src/embedding/embedder.js

@@ -0,0 +1,54 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Embedder = void 0;
+const openai_1 = __importDefault(require("openai"));
+class Embedder {
+    constructor(model) {
+        this.client = null;
+        this.model = model;
+        if (this.isOpenAIModel()) {
+            const apiKey = process.env.OPENAI_API_KEY;
+            if (!apiKey) {
+                throw new Error('OPENAI_API_KEY is not set in environment variables. ' +
+                    'Please add it to your .env file: OPENAI_API_KEY=sk-your-key-here');
+            }
+            this.client = new openai_1.default({
+                apiKey: apiKey
+            });
+        }
+    }
+    isOpenAIModel() {
+        return this.model.startsWith('openai:');
+    }
+    getModelName() {
+        return this.model.replace('openai:', '');
+    }
+    async embed(text) {
+        if (this.isOpenAIModel() && this.client) {
+            const response = await this.client.embeddings.create({
+                model: this.getModelName(),
+                input: text
+            });
+            return response.data[0].embedding;
+        }
+        throw new Error(`Unsupported embedding model: ${this.model}`);
+    }
+    async embedBatch(texts, batchSize = 100) {
+        const embeddings = [];
+        for (let i = 0; i < texts.length; i += batchSize) {
+            const batch = texts.slice(i, i + batchSize);
+            if (this.isOpenAIModel() && this.client) {
+                const response = await this.client.embeddings.create({
+                    model: this.getModelName(),
+                    input: batch
+                });
+                embeddings.push(...response.data.map(d => d.embedding));
+            }
+        }
+        return embeddings;
+    }
+}
+exports.Embedder = Embedder;

package/dist/src/embedding/pipeline.d.ts

@@ -0,0 +1,8 @@
+import { DataChunk, EmbeddingConfig } from '../types';
+export declare class EmbeddingPipeline {
+    private chunker;
+    private embedder;
+    private config;
+    constructor(config: EmbeddingConfig);
+    process(documents: any[]): Promise<DataChunk[]>;
+}

package/dist/src/embedding/pipeline.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EmbeddingPipeline = void 0;
+const chunker_1 = require("./chunker");
+const embedder_1 = require("./embedder");
+class EmbeddingPipeline {
+    constructor(config) {
+        this.config = config;
+        this.chunker = new chunker_1.TextChunker(config.chunkSize || 1000, config.chunkOverlap || 200);
+        this.embedder = new embedder_1.Embedder(config.embedding);
+    }
+    async process(documents) {
+        const chunked = this.chunker.chunkDocuments(documents);
+        const texts = chunked.map(doc => doc.content);
+        const embeddings = await this.embedder.embedBatch(texts, this.config.batchSize || 100);
+        return chunked.map((doc, index) => ({
+            id: `chunk_${index}`,
+            content: doc.content,
+            embedding: embeddings[index],
+            metadata: doc.metadata
+        }));
+    }
+}
+exports.EmbeddingPipeline = EmbeddingPipeline;

package/dist/src/index.d.ts

@@ -0,0 +1,3 @@
+export { ThinkBase } from './client';
+export { Dataset } from './dataset';
+export * from './types';

package/dist/src/index.js

@@ -0,0 +1,22 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Dataset = exports.ThinkBase = void 0;
+var client_1 = require("./client");
+Object.defineProperty(exports, "ThinkBase", { enumerable: true, get: function () { return client_1.ThinkBase; } });
+var dataset_1 = require("./dataset");
+Object.defineProperty(exports, "Dataset", { enumerable: true, get: function () { return dataset_1.Dataset; } });
+__exportStar(require("./types"), exports);

package/dist/src/query/natural-language.d.ts

@@ -0,0 +1,9 @@
+import { QueryResult } from '../types';
+export declare class NaturalLanguageQuery {
+    private client;
+    private schema;
+    constructor(schema?: any);
+    setSchema(schema: any): void;
+    translateToSQL(query: string, dialect?: string): Promise<string>;
+    executeQuery(query: string, executor: (sql: string) => Promise<any[]>): Promise<QueryResult>;
+}

package/dist/src/query/natural-language.js

@@ -0,0 +1,44 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.NaturalLanguageQuery = void 0;
+const openai_1 = __importDefault(require("openai"));
+class NaturalLanguageQuery {
+    constructor(schema) {
+        this.client = new openai_1.default({
+            apiKey: process.env.OPENAI_API_KEY
+        });
+        this.schema = schema;
+    }
+    setSchema(schema) {
+        this.schema = schema;
+    }
+    async translateToSQL(query, dialect = 'postgres') {
+        const schemaContext = this.schema
+            ? `Database schema:\n${JSON.stringify(this.schema, null, 2)}\n\n`
+            : '';
+        const prompt = `${schemaContext}Convert this natural language query to ${dialect} SQL:
+"${query}"
+
+Return ONLY the SQL query, no explanations.`;
+        const response = await this.client.chat.completions.create({
+            model: 'gpt-4o-mini',
+            messages: [{ role: 'user', content: prompt }],
+            temperature: 0
+        });
+        return response.choices[0].message.content?.trim() || '';
+    }
+    async executeQuery(query, executor) {
+        const sql = await this.translateToSQL(query);
+        try {
+            const data = await executor(sql);
+            return { data, sql };
+        }
+        catch (error) {
+            throw new Error(`Query execution failed: ${error.message}`);
+        }
+    }
+}
+exports.NaturalLanguageQuery = NaturalLanguageQuery;

package/dist/src/search/vector.d.ts

@@ -0,0 +1,8 @@
+import { DataChunk, SearchResult } from '../types';
+export declare class VectorSearch {
+    private chunks;
+    constructor(chunks: DataChunk[]);
+    cosineSimilarity(a: number[], b: number[]): number;
+    search(queryEmbedding: number[], limit?: number, threshold?: number): SearchResult[];
+    hybridSearch(queryEmbedding: number[], keywords: string[], limit?: number, vectorWeight?: number): SearchResult[];
+}

package/dist/src/search/vector.js

@@ -0,0 +1,65 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VectorSearch = void 0;
+class VectorSearch {
+    constructor(chunks) {
+        this.chunks = chunks;
+    }
+    cosineSimilarity(a, b) {
+        if (a.length !== b.length) {
+            throw new Error('Vectors must have the same length');
+        }
+        let dotProduct = 0;
+        let normA = 0;
+        let normB = 0;
+        for (let i = 0; i < a.length; i++) {
+            dotProduct += a[i] * b[i];
+            normA += a[i] * a[i];
+            normB += b[i] * b[i];
+        }
+        return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+    }
+    search(queryEmbedding, limit = 5, threshold = 0.7) {
+        const results = [];
+        for (const chunk of this.chunks) {
+            if (!chunk.embedding)
+                continue;
+            const score = this.cosineSimilarity(queryEmbedding, chunk.embedding);
+            if (score >= threshold) {
+                results.push({
+                    id: chunk.id,
+                    content: chunk.content,
+                    score,
+                    metadata: chunk.metadata
+                });
+            }
+        }
+        return results
+            .sort((a, b) => b.score - a.score)
+            .slice(0, limit);
+    }
+    hybridSearch(queryEmbedding, keywords, limit = 5, vectorWeight = 0.7) {
+        const results = [];
+        const keywordWeight = 1 - vectorWeight;
+        for (const chunk of this.chunks) {
+            if (!chunk.embedding)
+                continue;
+            const vectorScore = this.cosineSimilarity(queryEmbedding, chunk.embedding);
+            const lowerContent = chunk.content.toLowerCase();
+            const keywordScore = keywords.reduce((score, keyword) => {
+                return score + (lowerContent.includes(keyword.toLowerCase()) ? 1 : 0);
+            }, 0) / keywords.length;
+            const combinedScore = (vectorScore * vectorWeight) + (keywordScore * keywordWeight);
+            results.push({
+                id: chunk.id,
+                content: chunk.content,
+                score: combinedScore,
+                metadata: chunk.metadata
+            });
+        }
+        return results
+            .sort((a, b) => b.score - a.score)
+            .slice(0, limit);
+    }
+}
+exports.VectorSearch = VectorSearch;

package/dist/src/types.d.ts

@@ -0,0 +1,48 @@
+export interface ThinkBaseConfig {
+    apiKey?: string;
+    apiUrl?: string;
+}
+export type DataSourceType = 'postgres' | 'mysql' | 'mongodb' | 'rest' | 'graphql' | 'csv' | 'json' | 'webscrape';
+export interface DataSourceConfig {
+    source: DataSourceType;
+    connection?: string;
+    url?: string;
+    data?: any;
+    query?: string;
+    collection?: string;
+    table?: string;
+}
+export interface EmbeddingConfig {
+    embedding: string;
+    chunkSize?: number;
+    chunkOverlap?: number;
+    batchSize?: number;
+}
+export interface SearchOptions {
+    limit?: number;
+    threshold?: number;
+    filter?: Record<string, any>;
+    hybrid?: boolean;
+}
+export interface ContextOptions {
+    query: string;
+    limit?: number;
+    maxTokens?: number;
+}
+export interface DataChunk {
+    id: string;
+    content: string;
+    embedding?: number[];
+    metadata?: Record<string, any>;
+}
+export interface SearchResult {
+    id: string;
+    content: string;
+    score: number;
+    metadata?: Record<string, any>;
+}
+export interface QueryResult {
+    data: any[];
+    sql?: string;
+    metadata?: Record<string, any>;
+}

package/dist/src/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@thinkbase/sdk",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "description": "ThinkBase SDK - Turn any data into structured, searchable, and AI-ready context with a single API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -25,11 +25,12 @@
   "author": "ThinkBase",
   "license": "MIT",
   "dependencies": {
-    "openai": "^4.68.4",
-    "pg": "^8.13.1",
-    "mysql2": "^3.11.5",
+    "axios": "^1.7.9",
+    "dotenv": "^17.3.1",
     "mongodb": "^6.10.0",
-    "axios": "^1.7.9"
+    "mysql2": "^3.11.5",
+    "openai": "^4.68.4",
+    "pg": "^8.13.1"
   },
   "devDependencies": {
     "@types/node": "^22.10.2",