@loghead/core 0.1.18 → 0.1.20

package/src/api/server.ts

@@ -1,154 +0,0 @@
-import express from "express";
-import cors from "cors";
-import { DbService } from "../services/db";
-import { AuthService } from "../services/auth";
-import chalk from "chalk";
-
-const auth = new AuthService();
-
-export async function startApiServer(db: DbService) {
-    const app = express();
-    const port = process.env.PORT || 4567;
-
-    app.use(cors());
-    app.use(express.json());
-
-    await auth.initialize();
-
-    console.log(chalk.bold.green(`\n💻 MCP server running on:`));
-    console.log(chalk.green(`http://localhost:${port}`));
-
-    app.post("/api/ingest", async (req, res) => {
-        try {
-            const authHeader = req.headers.authorization;
-            if (!authHeader || !authHeader.startsWith("Bearer ")) {
-                return res.status(401).send("Unauthorized: Missing token");
-            }
-            const token = authHeader.split(" ")[1];
-            const payload = await auth.verifyToken(token);
-            if (!payload || !payload.streamId) {
-                return res.status(401).send("Unauthorized: Invalid token");
-            }
-
-            const { streamId, logs } = req.body;
-
-            if (streamId !== payload.streamId) {
-                return res.status(403).send("Forbidden: Token does not match streamId");
-            }
-
-            if (!logs) {
-                return res.status(400).send("Missing logs");
-            }
-
-            const logEntries = Array.isArray(logs) ? logs : [logs];
-
-            for (const log of logEntries) {
-                let content = "";
-                let metadata = {};
-
-                if (typeof log === "string") {
-                    content = log;
-                } else if (typeof log === "object") {
-                    content = log.content || JSON.stringify(log);
-                    metadata = log.metadata || {};
-                }
-
-                if (content) {
-                    await db.addLog(streamId, content, metadata);
-                }
-            }
-
-            res.json({ success: true, count: logEntries.length });
-        } catch (e) {
-            console.error("Ingest error:", e);
-            res.status(500).json({ error: String(e) });
-        }
-    });
-
-    app.get("/api/projects", (req, res) => {
-        const projects = db.listProjects();
-        res.json(projects);
-    });
-
-    app.post("/api/projects", (req, res) => {
-        const { name } = req.body;
-        if (!name) return res.status(400).json({ error: "Name required" });
-        const project = db.createProject(name);
-        res.json(project);
-    });
-
-    app.delete("/api/projects/:id", (req, res) => {
-        const { id } = req.params;
-        db.deleteProject(id);
-        res.json({ success: true });
-    });
-
-    app.get("/api/streams", (req, res) => {
-        const projectId = req.query.projectId as string;
-        if (projectId) {
-            const streams = db.listStreams(projectId);
-            res.json(streams);
-        } else {
-            res.status(400).send("Missing projectId");
-        }
-    });
-
-    app.delete("/api/streams/:id", (req, res) => {
-        const { id } = req.params;
-        db.deleteStream(id);
-        res.json({ success: true });
-    });
-
-    app.post("/api/streams", (req, res) => {
-        // Deprecated or just listing? The previous code had this returning listStreams for POST?
-        // I'll remove it or keep it if CLI uses it?
-        // CLI uses db directly.
-        // MCP uses GET /api/streams
-        // I'll replace this with the actual CREATE logic to be RESTful, or keep /create
-        const projectId = req.body.projectId;
-        if (projectId) {
-            const streams = db.listStreams(projectId);
-            res.json(streams);
-        } else {
-            res.status(400).send("Missing projectId");
-        }
-    });
-
-    app.post("/api/streams/create", async (req, res) => {
-        const body = req.body;
-        const stream = await db.createStream(body.projectId, body.type, body.name, body.config || {});
-        res.json(stream);
-    });
-
-    app.get("/api/logs", async (req, res) => {
-        const streamId = req.query.streamId as string;
-        if (!streamId) {
-            return res.status(400).send("Missing streamId");
-        }
-
-        let page = parseInt((req.query.page as string) || "1");
-        if (page < 1) page = 1;
-
-        let pageSize = parseInt((req.query.pageSize as string) || "100");
-        let limit = req.query.limit ? parseInt(req.query.limit as string) : pageSize;
-
-        // Enforce max limit
-        if (limit > 1000) limit = 1000;
-
-        const offset = (page - 1) * limit;
-
-        const query = req.query.q as string;
-
-        let logs;
-        if (query) {
-            logs = await db.searchLogs(streamId, query, limit);
-        } else {
-            logs = db.getRecentLogs(streamId, limit, offset);
-        }
-        res.json(logs);
-    });
-
-    app.listen(port, () => {
-        // listening
-    });
-}

package/src/cli_main.ts

@@ -1,83 +0,0 @@
-#!/usr/bin/env node
-import yargs from "yargs";
-import { hideBin } from "yargs/helpers";
-import { DbService } from "./services/db";
-import { startApiServer } from "./api/server";
-import { migrate } from "./db/migrate";
-// import { ensureInfrastructure } from "./utils/startup"; // Might need adjustment
-import { startTui } from "./ui/main";
-import { AuthService } from "./services/auth";
-import chalk from "chalk";
-
-const db = new DbService();
-const auth = new AuthService();
-
-async function main() {
-    const argv = await yargs(hideBin(process.argv))
-        .command(["start", "$0"], "Start API Server", {}, async () => {
-            console.log("Ensuring database is initialized...");
-            await migrate(false); // Run migrations silently
-
-            const token = await auth.getOrCreateMcpToken();
-
-            // Start API Server (this sets up express listen)
-            await startApiServer(db);
-
-            // Start TUI (this will clear screen and take over)
-            await startTui(db, token);
-            process.exit(0);
-        })
-        .command("projects <cmd> [name]", "Manage projects", (yargs) => {
-            yargs
-                .command("list", "List projects", {}, () => {
-                    const projects = db.listProjects();
-                    console.table(projects);
-                })
-                .command("add <name>", "Add project", {}, (argv) => {
-                    const p = db.createProject(argv.name as string);
-                    console.log(`Project created: ${p.id}`);
-                })
-                .command("delete <id>", "Delete project", {}, (argv) => {
-                    db.deleteProject(argv.id as string);
-                    console.log(`Project deleted: ${argv.id}`);
-                });
-        })
-        .command("streams <cmd> [type] [name]", "Manage streams", (yargs) => {
-            yargs
-                .command("list", "List streams", {
-                    project: { type: "string", demandOption: true }
-                }, (argv) => {
-                    const streams = db.listStreams(argv.project);
-                    console.table(streams);
-                })
-                .command("add <type> <name>", "Add stream", {
-                    project: { type: "string", demandOption: true },
-                    container: { type: "string" }
-                }, async (argv) => {
-                    const config: Record<string, unknown> = {};
-                    if (argv.type === "docker" && argv.container) {
-                        config.container = argv.container;
-                    }
-                    const s = await db.createStream(argv.project, argv.type as string, argv.name as string, config);
-                    console.log(`Stream created: ${s.id}`);
-                    console.log(`Token: ${s.token}`);
-                })
-                .command("token <streamId>", "Get token for stream", {}, async (argv) => {
-                    const token = await auth.createStreamToken(argv.streamId as string);
-                    console.log(`Token: ${token}`);
-                })
-                .command("delete <id>", "Delete stream", {}, (argv) => {
-                    db.deleteStream(argv.id as string);
-                    console.log(`Stream deleted: ${argv.id}`);
-                });
-        })
-        .demandCommand(1)
-        .strict()
-        .help()
-        .parse();
-}
-
-main().catch(err => {
-    console.error(err);
-    process.exit(1);
-});

package/src/db/client.ts

@@ -1,20 +0,0 @@
-import Database from "better-sqlite3";
-import * as sqliteVec from "sqlite-vec";
-import path from "path";
-import dotenv from "dotenv";
-
-dotenv.config();
-
-const dbPath = process.env.LOGHEAD_DB_PATH || "loghead.db";
-console.log(`[DB] Using database at: ${path.resolve(dbPath)}`);
-
-const db = new Database(dbPath);
-
-// Load sqlite-vec extension
-try {
-    sqliteVec.load(db);
-} catch (e) {
-    console.error("Failed to load sqlite-vec extension:", e);
-}
-
-export { db };

package/src/db/migrate.ts

@@ -1,67 +0,0 @@
-import { db } from "./client";
-
-export function migrate(verbose = true) {
-  if (verbose) console.log("Running migrations...");
-
-  // Enable foreign keys
-  db.exec("PRAGMA foreign_keys = ON;");
-
-  // System Config table (for secrets, etc.)
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS system_config (
-      key TEXT PRIMARY KEY,
-      value TEXT NOT NULL
-    );
-  `);
-
-  // Projects table
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS projects (
-      id TEXT PRIMARY KEY,
-      name TEXT NOT NULL,
-      created_at DATETIME DEFAULT CURRENT_TIMESTAMP
-    );
-  `);
-
-  // Data Streams table
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS data_streams (
-      id TEXT PRIMARY KEY,
-      project_id TEXT,
-      type TEXT NOT NULL,
-      name TEXT NOT NULL,
-      config TEXT DEFAULT '{}',
-      created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
-      FOREIGN KEY(project_id) REFERENCES projects(id) ON DELETE CASCADE
-    );
-  `);
-
-  // Logs table
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS logs (
-      id TEXT PRIMARY KEY,
-      stream_id TEXT,
-      content TEXT NOT NULL,
-      timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
-      metadata TEXT DEFAULT '{}',
-      FOREIGN KEY(stream_id) REFERENCES data_streams(id) ON DELETE CASCADE
-    );
-  `);
-
-  // Vector table (using sqlite-vec)
-  // Assuming 1024 dimensions for qwen3-embedding:0.6b (check actual dim)
-  // qwen2.5-0.5b is 1536?
-  // qwen-embedding-0.6b might be 384 or 1024?
-  // Let's assume 1024 as per previous code.
-  try {
-    db.exec(`
-      CREATE VIRTUAL TABLE IF NOT EXISTS vec_logs USING vec0(
-        embedding float[1024]
-      );
-    `);
-  } catch (e) {
-    console.warn("Failed to create virtual vector table. Is sqlite-vec loaded?", e);
-  }
-
-  if (verbose) console.log("Migrations complete.");
-}

package/src/services/auth.ts

@@ -1,74 +0,0 @@
-import jwt from "jsonwebtoken";
-import { db } from "../db/client";
-import { randomBytes } from "crypto";
-
-// Helper type for DB access
-// deno-lint-ignore no-explicit-any
-type DbAny = any;
-
-export class AuthService {
-    private secretKey: string | null = null;
-
-    async initialize() {
-        if (this.secretKey) return;
-
-        // Try to load secret from DB
-        const row = (db.prepare("SELECT value FROM system_config WHERE key = 'jwt_secret'") as unknown as DbAny).get();
-
-        let rawSecret = row?.value;
-
-        if (!rawSecret) {
-            // Generate new secret
-            rawSecret = randomBytes(64).toString('hex');
-            (db.prepare("INSERT INTO system_config (key, value) VALUES ('jwt_secret', ?)") as unknown as DbAny).run(rawSecret);
-        }
-
-        this.secretKey = rawSecret;
-    }
-
-    async getOrCreateMcpToken(): Promise<string> {
-        await this.initialize();
-        if (!this.secretKey) throw new Error("Auth not initialized");
-
-        // Check if token exists in DB
-        const row = (db.prepare("SELECT value FROM system_config WHERE key = 'mcp_token'") as unknown as DbAny).get();
-        if (row?.value) {
-            return row.value;
-        }
-
-        // Create new token
-        // A system token that has access to everything (conceptually)
-        const token = jwt.sign({ sub: "system:mcp", iss: "loghead", role: "admin" }, this.secretKey, { algorithm: "HS512" });
-
-        (db.prepare("INSERT INTO system_config (key, value) VALUES ('mcp_token', ?)") as unknown as DbAny).run(token);
-
-        return token;
-    }
-
-    async createStreamToken(streamId: string): Promise<string> {
-        await this.initialize();
-        if (!this.secretKey) throw new Error("Auth not initialized");
-
-        const token = jwt.sign({ sub: streamId, iss: "loghead" }, this.secretKey, { algorithm: "HS512" });
-        return token;
-    }
-
-    async verifyToken(token: string): Promise<{ streamId: string } | null> {
-        await this.initialize();
-        if (!this.secretKey) throw new Error("Auth not initialized");
-
-        try {
-            // console.log(`[Auth] Verifying token with secret: ${this.secretKey.substring(0, 10)}...`);
-            const payload = jwt.verify(token, this.secretKey, { issuer: "loghead", algorithms: ["HS512"] }) as jwt.JwtPayload;
-            if (!payload.sub) return null;
-            return { streamId: payload.sub };
-        } catch (e) {
-            console.error("Token verification failed:", e);
-            if (e instanceof Error && e.message === "invalid signature") {
-                console.error("[Auth] Secret key mismatch. Ensure the server is using the same database (and secret) as when the token was generated.");
-                console.error(`[Auth] Current secret starts with: ${this.secretKey?.substring(0, 8)}...`);
-            }
-            return null;
-        }
-    }
-}

package/src/services/db.ts

@@ -1,173 +0,0 @@
-import { db } from "../db/client";
-import { OllamaService } from "./ollama";
-import { AuthService } from "./auth";
-import { Project, Stream, Log, SearchResult } from "../types";
-import { randomUUID } from "crypto";
-
-const ollama = new OllamaService();
-const auth = new AuthService();
-
-// Helper to cast DB results to avoid no-explicit-any
-// deno-lint-ignore no-explicit-any
-type DbAny = any;
-
-export class DbService {
-    createProject(name: string): Project {
-        const id = randomUUID();
-        (db.prepare("INSERT INTO projects (id, name) VALUES (?, ?)") as unknown as DbAny).run(id, name);
-        return this.getProject(id);
-    }
-
-    getProject(id: string): Project {
-        return (db.prepare("SELECT * FROM projects WHERE id = ?") as unknown as DbAny).get(id);
-    }
-
-    deleteProject(id: string): boolean {
-        (db.prepare("DELETE FROM projects WHERE id = ?") as unknown as DbAny).run(id);
-        return true;
-    }
-
-    listProjects(): Project[] {
-        try {
-            const projects = (db.prepare("SELECT * FROM projects ORDER BY created_at DESC") as unknown as DbAny).all();
-            return projects.map((p: Project) => {
-                const streams = (db.prepare("SELECT * FROM data_streams WHERE project_id = ?") as unknown as DbAny).all(p.id);
-                return { ...p, streams };
-            });
-        } catch (e) {
-            console.error("Error in listProjects:", e);
-            throw e;
-        }
-    }
-
-    async createStream(projectId: string, type: string, name: string, config: Record<string, unknown> = {}): Promise<Stream & { token: string }> {
-        const id = randomUUID();
-        (db.prepare("INSERT INTO data_streams (id, project_id, type, name, config) VALUES (?, ?, ?, ?, ?)") as unknown as DbAny).run(
-            id, projectId, type, name, JSON.stringify(config)
-        );
-
-        const token = await auth.createStreamToken(id);
-        const stream = this.getStream(id);
-
-        return { ...stream, token };
-    }
-
-    getStream(id: string): Stream {
-        const stream = (db.prepare("SELECT * FROM data_streams WHERE id = ?") as unknown as DbAny).get(id);
-        if (stream && typeof stream.config === "string") {
-            try { stream.config = JSON.parse(stream.config); } catch { /* ignore */ }
-        }
-        return stream;
-    }
-
-    deleteStream(id: string): boolean {
-        (db.prepare("DELETE FROM data_streams WHERE id = ?") as unknown as DbAny).run(id);
-        return true;
-    }
-
-    listStreams(projectId: string): Stream[] {
-        const streams = (db.prepare("SELECT * FROM data_streams WHERE project_id = ? ORDER BY created_at DESC") as unknown as DbAny).all(projectId);
-        return streams.map((s: Stream) => {
-            if (typeof s.config === "string") try { s.config = JSON.parse(s.config); } catch { /* ignore */ }
-            return s;
-        });
-    }
-
-    async addLog(streamId: string, content: string, metadata: Record<string, unknown> = {}): Promise<{ id: string }> {
-        // Generate embedding
-        let embedding: number[] | null = null;
-        try {
-            embedding = await ollama.generateEmbedding(content);
-        } catch (_e) {
-            // console.warn("Embedding failed", _e);
-        }
-
-        const id = randomUUID();
-        const metadataStr = JSON.stringify(metadata);
-
-        // Manual Transaction
-        const insertTx = db.transaction(() => {
-            // 1. Insert into logs
-            const info = (db.prepare("INSERT INTO logs (id, stream_id, content, metadata) VALUES (?, ?, ?, ?)") as unknown as DbAny).run(
-                id, streamId, content, metadataStr
-            );
-
-            // 2. Get rowid directly
-            const rowid = info.lastInsertRowid;
-
-            // 3. Insert into vec_logs if embedding exists
-            if (embedding && embedding.length > 0) {
-                if (embedding.length !== 1024) {
-                    console.warn(`[Warning] Embedding dimension mismatch. Expected 1024, got ${embedding.length}. Skipping vector index.`);
-                    return;
-                }
-                const vectorJson = JSON.stringify(embedding);
-                // Explicitly cast rowid to BigInt to ensure better-sqlite3 binds it as an INTEGER
-                (db.prepare("INSERT INTO vec_logs(rowid, embedding) VALUES (?, ?)") as unknown as DbAny).run(BigInt(rowid), vectorJson);
-            }
-        });
-
-        try {
-            insertTx();
-        } catch (e) {
-            throw e;
-        }
-
-        return { id };
-    }
-
-    async searchLogs(streamId: string, query: string, limit = 10): Promise<SearchResult[]> {
-        const embedding = await ollama.generateEmbedding(query);
-        if (!embedding) return [];
-
-        const vectorJson = JSON.stringify(embedding);
-
-        // KNN Search
-        const rows = (db.prepare(`
-      SELECT l.content, l.timestamp, l.metadata, v.distance
-      FROM vec_logs v
-      JOIN logs l ON l.rowid = v.rowid
-      WHERE v.embedding MATCH ? AND k = ? AND l.stream_id = ?
-      ORDER BY v.distance
-    `) as unknown as DbAny).all(vectorJson, limit, streamId);
-
-        return rows.map((row: { content: string; timestamp: string; metadata: string; distance: number }) => {
-            let meta: Record<string, unknown> | undefined;
-            try { meta = JSON.parse(row.metadata); } catch { /* ignore */ }
-            return {
-                content: row.content,
-                timestamp: row.timestamp,
-                similarity: 1 - row.distance, // Rough approx
-                metadata: (meta && Object.keys(meta).length > 0) ? meta : undefined
-            };
-        });
-    }
-
-    getRecentLogs(streamId: string, limit = 50, offset = 0): Log[] {
-        const rows = (db.prepare(`
-      SELECT content, timestamp, metadata FROM logs
-      WHERE stream_id = ?
-      ORDER BY timestamp DESC
-      LIMIT ? OFFSET ?
-    `) as unknown as DbAny).all(streamId, limit, offset);
-
-        return rows.map((row: Log) => {
-            let meta = row.metadata;
-            if (typeof meta === "string") {
-                try { meta = JSON.parse(meta); } catch { /* ignore */ }
-            }
-            return {
-                id: row.id, // Ensure id is included if needed, or update Log type
-                stream_id: streamId,
-                content: row.content,
-                timestamp: row.timestamp,
-                metadata: (typeof meta === "object" && meta && Object.keys(meta).length > 0) ? meta : {}
-            } as Log;
-        });
-    }
-
-    close() {
-        db.close();
-    }
-}
-

package/src/services/ollama.ts

@@ -1,39 +0,0 @@
-import { Ollama } from "ollama";
-
-export class OllamaService {
-    private client: Ollama;
-    private model: string;
-
-    constructor(host = "http://localhost:11434", model = "qwen3-embedding:0.6b") {
-        this.client = new Ollama({ host });
-        this.model = model;
-    }
-
-    async generateEmbedding(prompt: string): Promise<number[]> {
-        try {
-            const response = await this.client.embeddings({
-                model: this.model,
-                prompt: prompt,
-            });
-            return response.embedding;
-        } catch (error) {
-            console.error("Failed to generate embedding:", error);
-            throw error;
-        }
-    }
-
-    async ensureModel() {
-        try {
-            const list = await this.client.list();
-            const exists = list.models.some((m) => m.name.includes(this.model));
-
-            if (!exists) {
-                console.log(`Model ${this.model} not found. Pulling...`);
-                await this.client.pull({ model: this.model });
-                console.log("Model pulled.");
-            }
-        } catch (e) {
-            console.warn("Could not check/pull ollama model:", e);
-        }
-    }
-}

package/src/tests/db.test.ts

@@ -1,63 +0,0 @@
-import { assertEquals, assertExists } from "@std/assert";
-import { migrate } from "../db/migrate.ts";
-
-// Set up environment to use in-memory DB before importing client
-Deno.env.set("LOGHEAD_DB_PATH", ":memory:");
-
-// Dynamic import to ensure env var is set before module loads
-// We trigger the side-effect of client.ts (creating DB connection)
-await import("../db/client.ts");
-const { DbService } = await import("../services/db.ts");
-
-Deno.test("DbService - Project Management", async (t) => {
-    // @ts-ignore: Dynamic import returns any
-    const service = new DbService();
-
-    // Initialize DB schema
-    migrate(false);
-
-    await t.step("createProject creates a project", () => {
-        const project = service.createProject("Test Project");
-        assertExists(project.id);
-        assertEquals(project.name, "Test Project");
-    });
-
-    await t.step("listProjects returns projects", () => {
-        const projects = service.listProjects();
-        assertExists(projects.find(p => p.name === "Test Project"));
-    });
-
-    await t.step("deleteProject deletes a project", () => {
-        const projects = service.listProjects();
-        const project = projects.find(p => p.name === "Test Project");
-        if (project) {
-            service.deleteProject(project.id);
-
-            const remaining = service.listProjects();
-            assertEquals(remaining.find(p => p.name === "Test Project"), undefined);
-        }
-    });
-});
-
-Deno.test("DbService - Stream Management", async (t) => {
-    // @ts-ignore: Dynamic import returns any
-    const service = new DbService();
-
-    // Create a fresh project for this test suite
-    const project = service.createProject("Stream Project");
-
-    await t.step("createStream creates a stream", () => {
-        const stream = service.createStream(project.id, "terminal", "My Stream", { foo: "bar" });
-        assertExists(stream.id);
-        assertEquals(stream.name, "My Stream");
-        assertEquals(stream.type, "terminal");
-        // @ts-ignore: Property access
-        assertEquals(stream.config, { foo: "bar" });
-    });
-
-    await t.step("listStreams returns streams", () => {
-        const streams = service.listStreams(project.id);
-        assertEquals(streams.length, 1);
-        assertEquals(streams[0].name, "My Stream");
-    });
-});