@loghead/core 0.1.1 → 0.1.3

package/package.json

@@ -1,32 +1,36 @@
 {
   "name": "@loghead/core",
-  "version": "0.1.1",
-  "description": "Smart log aggregation tool and MCP server",
+  "version": "0.1.3",
+  "description": "Core API and Database for Loghead",
+  "main": "dist/index.js",
   "bin": {
-    "loghead": "./bin/loghead"
+    "loghead": "./dist/cli_main.js"
   },
-  "files": [
-    "bin"
-  ],
-  "os": [
-    "darwin"
-  ],
-  "cpu": [
-    "arm64"
-  ],
   "scripts": {
-    "postinstall": "echo 'Note: This package contains a prebuilt binary for macOS ARM64.'"
+    "start": "tsx src/cli_main.ts start",
+    "ui": "tsx src/cli_main.ts ui",
+    "build": "tsc",
+    "dev": "tsx watch src/cli_main.ts",
+    "deploy": "npm publish --access public"
   },
-  "author": "Onvo AI",
-  "license": "MIT",
-  "publishConfig": {
-    "access": "public"
+  "dependencies": {
+    "better-sqlite3": "^9.4.0",
+    "express": "^4.18.2",
+    "cors": "^2.8.5",
+    "dotenv": "^16.4.1",
+    "yargs": "^17.7.2",
+    "jsonwebtoken": "^9.0.2",
+    "chalk": "^5.3.0",
+    "inquirer": "^9.2.14",
+    "cli-table3": "^0.6.3",
+    "ollama": "^0.5.9"
   },
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/onvo-ai/loghead.git"
-  },
-  "bugs": {
-    "url": "https://github.com/onvo-ai/loghead/issues"
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.9",
+    "@types/express": "^4.17.21",
+    "@types/cors": "^2.8.17",
+    "@types/yargs": "^17.0.32",
+    "@types/jsonwebtoken": "^9.0.5",
+    "@types/inquirer": "^9.0.7"
   }
-}
+}

package/src/api/server.ts

@@ -0,0 +1,123 @@
+import express from "express";
+import cors from "cors";
+import { DbService } from "../services/db";
+import { AuthService } from "../services/auth";
+import chalk from "chalk";
+
+const auth = new AuthService();
+
+export async function startApiServer(db: DbService) {
+    const app = express();
+    const port = process.env.PORT || 4567;
+
+    app.use(cors());
+    app.use(express.json());
+
+    await auth.initialize();
+
+    console.log(chalk.bold.green(`✔ Loghead Core API Server running on http://localhost:${port}`));
+
+    app.post("/api/ingest", async (req, res) => {
+        try {
+            const authHeader = req.headers.authorization;
+            if (!authHeader || !authHeader.startsWith("Bearer ")) {
+                return res.status(401).send("Unauthorized: Missing token");
+            }
+            const token = authHeader.split(" ")[1];
+            const payload = await auth.verifyToken(token);
+            if (!payload || !payload.streamId) {
+                return res.status(401).send("Unauthorized: Invalid token");
+            }
+
+            const { streamId, logs } = req.body;
+
+            if (streamId !== payload.streamId) {
+                return res.status(403).send("Forbidden: Token does not match streamId");
+            }
+
+            if (!logs) {
+                return res.status(400).send("Missing logs");
+            }
+
+            const logEntries = Array.isArray(logs) ? logs : [logs];
+
+            for (const log of logEntries) {
+                let content = "";
+                let metadata = {};
+
+                if (typeof log === "string") {
+                    content = log;
+                } else if (typeof log === "object") {
+                    content = log.content || JSON.stringify(log);
+                    metadata = log.metadata || {};
+                }
+
+                if (content) {
+                    await db.addLog(streamId, content, metadata);
+                }
+            }
+
+            res.json({ success: true, count: logEntries.length });
+        } catch (e) {
+            console.error("Ingest error:", e);
+            res.status(500).json({ error: String(e) });
+        }
+    });
+
+    app.get("/api/projects", (req, res) => {
+        const projects = db.listProjects();
+        res.json(projects);
+    });
+
+    app.post("/api/projects", (req, res) => {
+        const projects = db.listProjects();
+        res.json(projects);
+    });
+
+    app.get("/api/streams", (req, res) => {
+        const projectId = req.query.projectId as string;
+        if (projectId) {
+            const streams = db.listStreams(projectId);
+            res.json(streams);
+        } else {
+            res.status(400).send("Missing projectId");
+        }
+    });
+
+    app.post("/api/streams", (req, res) => {
+        const projectId = req.body.projectId;
+        if (projectId) {
+            const streams = db.listStreams(projectId);
+            res.json(streams);
+        } else {
+            res.status(400).send("Missing projectId");
+        }
+    });
+
+    app.post("/api/streams/create", async (req, res) => {
+        const body = req.body;
+        const stream = await db.createStream(body.projectId, body.type, body.name, {});
+        res.json(stream);
+    });
+
+    app.get("/api/logs", async (req, res) => {
+        const streamId = req.query.streamId as string;
+        if (!streamId) {
+            return res.status(400).send("Missing streamId");
+        }
+        const limit = parseInt((req.query.limit as string) || "50");
+        const query = req.query.q as string;
+
+        let logs;
+        if (query) {
+            logs = await db.searchLogs(streamId, query, limit);
+        } else {
+            logs = db.getRecentLogs(streamId, limit);
+        }
+        res.json(logs);
+    });
+
+    app.listen(port, () => {
+        // listening
+    });
+}

package/src/cli_main.ts

@@ -0,0 +1,66 @@
+#!/usr/bin/env node
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import { DbService } from "./services/db";
+import { startApiServer } from "./api/server";
+import { migrate } from "./db/migrate";
+// import { ensureInfrastructure } from "./utils/startup"; // Might need adjustment
+import { startTui } from "./ui/main";
+
+const db = new DbService();
+
+async function main() {
+    const argv = await yargs(hideBin(process.argv))
+        .command("init", "Initialize/Migrate database", {}, async () => {
+            console.log("Initializing database...");
+            await migrate();
+        })
+        .command("start", "Start API Server", {}, async () => {
+            // await ensureInfrastructure();
+            await startApiServer(db);
+        })
+        .command("ui", "Start Terminal UI", {}, async () => {
+            await startTui(db);
+        })
+        .command("projects <cmd> [name]", "Manage projects", (yargs) => {
+            yargs
+                .command("list", "List projects", {}, () => {
+                    const projects = db.listProjects();
+                    console.table(projects);
+                })
+                .command("add <name>", "Add project", {}, (argv) => {
+                    const p = db.createProject(argv.name as string);
+                    console.log(`Project created: ${p.id}`);
+                });
+        })
+        .command("streams <cmd> [type] [name]", "Manage streams", (yargs) => {
+            yargs
+                .command("list", "List streams", {
+                    project: { type: "string", demandOption: true }
+                }, (argv) => {
+                    const streams = db.listStreams(argv.project);
+                    console.table(streams);
+                })
+                .command("add <type> <name>", "Add stream", {
+                    project: { type: "string", demandOption: true },
+                    container: { type: "string" }
+                }, async (argv) => {
+                    const config: Record<string, unknown> = {};
+                    if (argv.type === "docker" && argv.container) {
+                        config.container = argv.container;
+                    }
+                    const s = await db.createStream(argv.project, argv.type as string, argv.name as string, config);
+                    console.log(`Stream created: ${s.id}`);
+                    console.log(`Token: ${s.token}`);
+                });
+        })
+        .demandCommand(1)
+        .strict()
+        .help()
+        .parse();
+}
+
+main().catch(err => {
+    console.error(err);
+    process.exit(1);
+});

package/src/db/client.ts

@@ -0,0 +1,19 @@
+import Database from "better-sqlite3";
+import * as sqliteVec from "sqlite-vec";
+import path from "path";
+import dotenv from "dotenv";
+
+dotenv.config();
+
+const dbPath = process.env.LOGHEAD_DB_PATH || "loghead.db";
+
+const db = new Database(dbPath);
+
+// Load sqlite-vec extension
+try {
+    sqliteVec.load(db);
+} catch (e) {
+    console.error("Failed to load sqlite-vec extension:", e);
+}
+
+export { db };

package/src/db/migrate.ts

@@ -0,0 +1,67 @@
+import { db } from "./client";
+
+export function migrate(verbose = true) {
+  if (verbose) console.log("Running migrations...");
+
+  // Enable foreign keys
+  db.exec("PRAGMA foreign_keys = ON;");
+
+  // System Config table (for secrets, etc.)
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS system_config (
+      key TEXT PRIMARY KEY,
+      value TEXT NOT NULL
+    );
+  `);
+
+  // Projects table
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS projects (
+      id TEXT PRIMARY KEY,
+      name TEXT NOT NULL,
+      created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+    );
+  `);
+
+  // Data Streams table
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS data_streams (
+      id TEXT PRIMARY KEY,
+      project_id TEXT,
+      type TEXT NOT NULL,
+      name TEXT NOT NULL,
+      config TEXT DEFAULT '{}',
+      created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+      FOREIGN KEY(project_id) REFERENCES projects(id) ON DELETE CASCADE
+    );
+  `);
+
+  // Logs table
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS logs (
+      id TEXT PRIMARY KEY,
+      stream_id TEXT,
+      content TEXT NOT NULL,
+      timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
+      metadata TEXT DEFAULT '{}',
+      FOREIGN KEY(stream_id) REFERENCES data_streams(id) ON DELETE CASCADE
+    );
+  `);
+
+  // Vector table (using sqlite-vec)
+  // Assuming 1024 dimensions for qwen3-embedding:0.6b (check actual dim)
+  // qwen2.5-0.5b is 1536?
+  // qwen-embedding-0.6b might be 384 or 1024?
+  // Let's assume 1024 as per previous code.
+  try {
+    db.exec(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS vec_logs USING vec0(
+        embedding float[1024]
+      );
+    `);
+  } catch (e) {
+    console.warn("Failed to create virtual vector table. Is sqlite-vec loaded?", e);
+  }
+
+  if (verbose) console.log("Migrations complete.");
+}

package/src/services/auth.ts

@@ -0,0 +1,50 @@
+import jwt from "jsonwebtoken";
+import { db } from "../db/client";
+import { randomBytes } from "crypto";
+
+// Helper type for DB access
+// deno-lint-ignore no-explicit-any
+type DbAny = any;
+
+export class AuthService {
+    private secretKey: string | null = null;
+
+    async initialize() {
+        if (this.secretKey) return;
+
+        // Try to load secret from DB
+        const row = (db.prepare("SELECT value FROM system_config WHERE key = 'jwt_secret'") as unknown as DbAny).get();
+
+        let rawSecret = row?.value;
+
+        if (!rawSecret) {
+            // Generate new secret
+            rawSecret = randomBytes(64).toString('hex');
+            (db.prepare("INSERT INTO system_config (key, value) VALUES ('jwt_secret', ?)") as unknown as DbAny).run(rawSecret);
+        }
+
+        this.secretKey = rawSecret;
+    }
+
+    async createStreamToken(streamId: string): Promise<string> {
+        await this.initialize();
+        if (!this.secretKey) throw new Error("Auth not initialized");
+
+        const token = jwt.sign({ sub: streamId, iss: "loghead" }, this.secretKey, { algorithm: "HS512" });
+        return token;
+    }
+
+    async verifyToken(token: string): Promise<{ streamId: string } | null> {
+        await this.initialize();
+        if (!this.secretKey) throw new Error("Auth not initialized");
+
+        try {
+            const payload = jwt.verify(token, this.secretKey, { issuer: "loghead", algorithms: ["HS512"] }) as jwt.JwtPayload;
+            if (!payload.sub) return null;
+            return { streamId: payload.sub };
+        } catch (e) {
+            console.error("Token verification failed:", e);
+            return null;
+        }
+    }
+}

package/src/services/db.ts

@@ -0,0 +1,171 @@
+import { db } from "../db/client";
+import { OllamaService } from "./ollama";
+import { AuthService } from "./auth";
+import { Project, Stream, Log, SearchResult } from "../types";
+import { randomUUID } from "crypto";
+
+const ollama = new OllamaService();
+const auth = new AuthService();
+
+// Helper to cast DB results to avoid no-explicit-any
+// deno-lint-ignore no-explicit-any
+type DbAny = any;
+
+export class DbService {
+    createProject(name: string): Project {
+        const id = randomUUID();
+        (db.prepare("INSERT INTO projects (id, name) VALUES (?, ?)") as unknown as DbAny).run(id, name);
+        return this.getProject(id);
+    }
+
+    getProject(id: string): Project {
+        return (db.prepare("SELECT * FROM projects WHERE id = ?") as unknown as DbAny).get(id);
+    }
+
+    deleteProject(id: string): boolean {
+        (db.prepare("DELETE FROM projects WHERE id = ?") as unknown as DbAny).run(id);
+        return true;
+    }
+
+    listProjects(): Project[] {
+        console.error("Listing projects...");
+        try {
+            const projects = (db.prepare("SELECT * FROM projects ORDER BY created_at DESC") as unknown as DbAny).all();
+            console.error(`Found ${projects.length} projects.`);
+            return projects.map((p: Project) => {
+                const streams = (db.prepare("SELECT * FROM data_streams WHERE project_id = ?") as unknown as DbAny).all(p.id);
+                return { ...p, streams };
+            });
+        } catch (e) {
+            console.error("Error in listProjects:", e);
+            throw e;
+        }
+    }
+
+    async createStream(projectId: string, type: string, name: string, config: Record<string, unknown> = {}): Promise<Stream & { token: string }> {
+        const id = randomUUID();
+        (db.prepare("INSERT INTO data_streams (id, project_id, type, name, config) VALUES (?, ?, ?, ?, ?)") as unknown as DbAny).run(
+            id, projectId, type, name, JSON.stringify(config)
+        );
+
+        const token = await auth.createStreamToken(id);
+        const stream = this.getStream(id);
+
+        return { ...stream, token };
+    }
+
+    getStream(id: string): Stream {
+        const stream = (db.prepare("SELECT * FROM data_streams WHERE id = ?") as unknown as DbAny).get(id);
+        if (stream && typeof stream.config === "string") {
+            try { stream.config = JSON.parse(stream.config); } catch { /* ignore */ }
+        }
+        return stream;
+    }
+
+    deleteStream(id: string): boolean {
+        (db.prepare("DELETE FROM data_streams WHERE id = ?") as unknown as DbAny).run(id);
+        return true;
+    }
+
+    listStreams(projectId: string): Stream[] {
+        const streams = (db.prepare("SELECT * FROM data_streams WHERE project_id = ? ORDER BY created_at DESC") as unknown as DbAny).all(projectId);
+        return streams.map((s: Stream) => {
+            if (typeof s.config === "string") try { s.config = JSON.parse(s.config); } catch { /* ignore */ }
+            return s;
+        });
+    }
+
+    async addLog(streamId: string, content: string, metadata: Record<string, unknown> = {}): Promise<{ id: string }> {
+        // Generate embedding
+        let embedding: number[] | null = null;
+        try {
+            embedding = await ollama.generateEmbedding(content);
+        } catch (_e) {
+            // console.warn("Embedding failed", _e);
+        }
+
+        const id = randomUUID();
+        const metadataStr = JSON.stringify(metadata);
+
+        // Manual Transaction
+        const insertTx = db.transaction(() => {
+            // 1. Insert into logs
+            (db.prepare("INSERT INTO logs (id, stream_id, content, metadata) VALUES (?, ?, ?, ?)") as unknown as DbAny).run(
+                id, streamId, content, metadataStr
+            );
+
+            // 2. Get rowid
+            const rowInfo = (db.prepare("SELECT last_insert_rowid() as rowid") as unknown as DbAny).get();
+            const rowid = rowInfo.rowid;
+
+            // 3. Insert into vec_logs if embedding exists
+            if (embedding && embedding.length > 0) {
+                const vectorJson = JSON.stringify(embedding);
+                (db.prepare("INSERT INTO vec_logs(rowid, embedding) VALUES (?, ?)") as unknown as DbAny).run(rowid, vectorJson);
+            }
+        });
+
+        try {
+            insertTx();
+        } catch (e) {
+            throw e;
+        }
+
+        return { id };
+    }
+
+    async searchLogs(streamId: string, query: string, limit = 10): Promise<SearchResult[]> {
+        const embedding = await ollama.generateEmbedding(query);
+        if (!embedding) return [];
+
+        const vectorJson = JSON.stringify(embedding);
+
+        // KNN Search
+        const rows = (db.prepare(`
+      SELECT l.content, l.timestamp, l.metadata, v.distance
+      FROM vec_logs v
+      JOIN logs l ON l.rowid = v.rowid
+      WHERE v.embedding MATCH ? AND k = ? AND l.stream_id = ?
+      ORDER BY v.distance
+    `) as unknown as DbAny).all(vectorJson, limit, streamId);
+
+        return rows.map((row: { content: string; timestamp: string; metadata: string; distance: number }) => {
+            let meta: Record<string, unknown> | undefined;
+            try { meta = JSON.parse(row.metadata); } catch { /* ignore */ }
+            return {
+                content: row.content,
+                timestamp: row.timestamp,
+                similarity: 1 - row.distance, // Rough approx
+                metadata: (meta && Object.keys(meta).length > 0) ? meta : undefined
+            };
+        });
+    }
+
+    getRecentLogs(streamId: string, limit = 50): Log[] {
+        const rows = (db.prepare(`
+      SELECT content, timestamp, metadata FROM logs
+      WHERE stream_id = ?
+      ORDER BY timestamp DESC
+      LIMIT ?
+    `) as unknown as DbAny).all(streamId, limit);
+
+        return rows.map((row: Log) => {
+            let meta = row.metadata;
+            if (typeof meta === "string") {
+                try { meta = JSON.parse(meta); } catch { /* ignore */ }
+            }
+            return {
+                id: row.id, // Ensure id is included if needed, or update Log type
+                stream_id: streamId,
+                content: row.content,
+                timestamp: row.timestamp,
+                metadata: (typeof meta === "object" && meta && Object.keys(meta).length > 0) ? meta : {}
+            } as Log;
+        });
+    }
+
+    close() {
+        db.close();
+    }
+}
+

package/src/services/ollama.ts

@@ -0,0 +1,39 @@
+import { Ollama } from "ollama";
+
+export class OllamaService {
+    private client: Ollama;
+    private model: string;
+
+    constructor(host = "http://localhost:11434", model = "qwen3-embedding:0.6b") {
+        this.client = new Ollama({ host });
+        this.model = model;
+    }
+
+    async generateEmbedding(prompt: string): Promise<number[]> {
+        try {
+            const response = await this.client.embeddings({
+                model: this.model,
+                prompt: prompt,
+            });
+            return response.embedding;
+        } catch (error) {
+            console.error("Failed to generate embedding:", error);
+            throw error;
+        }
+    }
+
+    async ensureModel() {
+        try {
+            const list = await this.client.list();
+            const exists = list.models.some((m) => m.name.includes(this.model));
+
+            if (!exists) {
+                console.log(`Model ${this.model} not found. Pulling...`);
+                await this.client.pull({ model: this.model });
+                console.log("Model pulled.");
+            }
+        } catch (e) {
+            console.warn("Could not check/pull ollama model:", e);
+        }
+    }
+}

package/src/tests/db.test.ts

@@ -0,0 +1,63 @@
+import { assertEquals, assertExists } from "@std/assert";
+import { migrate } from "../db/migrate.ts";
+
+// Set up environment to use in-memory DB before importing client
+Deno.env.set("LOGHEAD_DB_PATH", ":memory:");
+
+// Dynamic import to ensure env var is set before module loads
+// We trigger the side-effect of client.ts (creating DB connection)
+await import("../db/client.ts");
+const { DbService } = await import("../services/db.ts");
+
+Deno.test("DbService - Project Management", async (t) => {
+    // @ts-ignore: Dynamic import returns any
+    const service = new DbService();
+
+    // Initialize DB schema
+    migrate(false);
+
+    await t.step("createProject creates a project", () => {
+        const project = service.createProject("Test Project");
+        assertExists(project.id);
+        assertEquals(project.name, "Test Project");
+    });
+
+    await t.step("listProjects returns projects", () => {
+        const projects = service.listProjects();
+        assertExists(projects.find(p => p.name === "Test Project"));
+    });
+
+    await t.step("deleteProject deletes a project", () => {
+        const projects = service.listProjects();
+        const project = projects.find(p => p.name === "Test Project");
+        if (project) {
+            service.deleteProject(project.id);
+
+            const remaining = service.listProjects();
+            assertEquals(remaining.find(p => p.name === "Test Project"), undefined);
+        }
+    });
+});
+
+Deno.test("DbService - Stream Management", async (t) => {
+    // @ts-ignore: Dynamic import returns any
+    const service = new DbService();
+
+    // Create a fresh project for this test suite
+    const project = service.createProject("Stream Project");
+
+    await t.step("createStream creates a stream", () => {
+        const stream = service.createStream(project.id, "terminal", "My Stream", { foo: "bar" });
+        assertExists(stream.id);
+        assertEquals(stream.name, "My Stream");
+        assertEquals(stream.type, "terminal");
+        // @ts-ignore: Property access
+        assertEquals(stream.config, { foo: "bar" });
+    });
+
+    await t.step("listStreams returns streams", () => {
+        const streams = service.listStreams(project.id);
+        assertEquals(streams.length, 1);
+        assertEquals(streams[0].name, "My Stream");
+    });
+});