@hasna/logs 0.0.1 → 0.1.0

package/src/lib/issues.test.ts

@@ -0,0 +1,79 @@
+import { describe, expect, it } from "bun:test"
+import { createTestDb } from "../db/index.ts"
+import { computeFingerprint, getIssue, listIssues, updateIssueStatus, upsertIssue } from "./issues.ts"
+
+function seedProject(db: ReturnType<typeof createTestDb>) {
+  return db.prepare("INSERT INTO projects (name) VALUES ('app') RETURNING id").get() as { id: string }
+}
+
+describe("computeFingerprint", () => {
+  it("returns consistent hash for same input", () => {
+    const a = computeFingerprint("error", "api", "DB connection failed")
+    const b = computeFingerprint("error", "api", "DB connection failed")
+    expect(a).toBe(b)
+  })
+
+  it("returns different hash for different messages", () => {
+    const a = computeFingerprint("error", "api", "timeout")
+    const b = computeFingerprint("error", "api", "DB error")
+    expect(a).not.toBe(b)
+  })
+
+  it("normalizes hex IDs in messages", () => {
+    const a = computeFingerprint("error", "api", "Error for id abc123def456")
+    const b = computeFingerprint("error", "api", "Error for id 000fffaabbcc")
+    expect(a).toBe(b)
+  })
+})
+
+describe("upsertIssue", () => {
+  it("creates a new issue", () => {
+    const db = createTestDb()
+    const p = seedProject(db)
+    const issue = upsertIssue(db, { project_id: p.id, level: "error", service: "api", message: "DB timeout" })
+    expect(issue.id).toBeTruthy()
+    expect(issue.count).toBe(1)
+    expect(issue.status).toBe("open")
+  })
+
+  it("increments count on duplicate", () => {
+    const db = createTestDb()
+    const p = seedProject(db)
+    upsertIssue(db, { project_id: p.id, level: "error", message: "Same error" })
+    upsertIssue(db, { project_id: p.id, level: "error", message: "Same error" })
+    const issue = upsertIssue(db, { project_id: p.id, level: "error", message: "Same error" })
+    expect(issue.count).toBe(3)
+  })
+
+  it("reopens resolved issues", () => {
+    const db = createTestDb()
+    const p = seedProject(db)
+    const issue = upsertIssue(db, { project_id: p.id, level: "error", message: "err" })
+    updateIssueStatus(db, issue.id, "resolved")
+    const reopened = upsertIssue(db, { project_id: p.id, level: "error", message: "err" })
+    expect(reopened.status).toBe("open")
+  })
+})
+
+describe("listIssues", () => {
+  it("filters by project and status", () => {
+    const db = createTestDb()
+    const p = seedProject(db)
+    const issue = upsertIssue(db, { project_id: p.id, level: "error", message: "database connection timed out" })
+    updateIssueStatus(db, issue.id, "resolved")
+    upsertIssue(db, { project_id: p.id, level: "error", message: "authentication service unavailable" })
+    expect(listIssues(db, p.id, "open")).toHaveLength(1)
+    expect(listIssues(db, p.id, "resolved")).toHaveLength(1)
+    expect(listIssues(db, p.id)).toHaveLength(2)
+  })
+})
+
+describe("updateIssueStatus", () => {
+  it("updates status", () => {
+    const db = createTestDb()
+    const p = seedProject(db)
+    const issue = upsertIssue(db, { project_id: p.id, level: "error", message: "x" })
+    const updated = updateIssueStatus(db, issue.id, "ignored")
+    expect(updated?.status).toBe("ignored")
+  })
+})

package/src/lib/issues.ts

@@ -0,0 +1,70 @@
+import type { Database } from "bun:sqlite"
+import { createHash } from "node:crypto"
+
+export interface Issue {
+  id: string
+  project_id: string | null
+  fingerprint: string
+  level: string
+  service: string | null
+  message_template: string
+  first_seen: string
+  last_seen: string
+  count: number
+  status: "open" | "resolved" | "ignored"
+}
+
+export function computeFingerprint(level: string, service: string | null, message: string, stackTrace?: string | null): string {
+  // Normalize message: strip hex IDs, numbers, timestamps
+  const normalized = message
+    .replace(/[0-9a-f]{8,}/gi, "<id>")
+    .replace(/\d+/g, "<n>")
+    .replace(/https?:\/\/[^\s]+/g, "<url>")
+    .trim()
+  const stackFrame = stackTrace ? stackTrace.split("\n").slice(0, 3).join("|") : ""
+  const raw = `${level}|${service ?? ""}|${normalized}|${stackFrame}`
+  return createHash("sha256").update(raw).digest("hex").slice(0, 16)
+}
+
+export function upsertIssue(db: Database, data: {
+  project_id?: string
+  level: string
+  service?: string | null
+  message: string
+  stack_trace?: string | null
+}): Issue {
+  const fingerprint = computeFingerprint(data.level, data.service ?? null, data.message, data.stack_trace)
+  return db.prepare(`
+    INSERT INTO issues (project_id, fingerprint, level, service, message_template)
+    VALUES ($project_id, $fingerprint, $level, $service, $message_template)
+    ON CONFLICT(project_id, fingerprint) DO UPDATE SET
+      count = count + 1,
+      last_seen = strftime('%Y-%m-%dT%H:%M:%fZ','now'),
+      status = CASE WHEN status = 'resolved' THEN 'open' ELSE status END
+    RETURNING *
+  `).get({
+    $project_id: data.project_id ?? null,
+    $fingerprint: fingerprint,
+    $level: data.level,
+    $service: data.service ?? null,
+    $message_template: data.message.slice(0, 500),
+  }) as Issue
+}
+
+export function listIssues(db: Database, projectId?: string, status?: string, limit = 50): Issue[] {
+  const conditions: string[] = []
+  const params: Record<string, unknown> = { $limit: limit }
+  if (projectId) { conditions.push("project_id = $p"); params.$p = projectId }
+  if (status) { conditions.push("status = $status"); params.$status = status }
+  const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : ""
+  return db.prepare(`SELECT * FROM issues ${where} ORDER BY last_seen DESC LIMIT $limit`).all(params) as Issue[]
+}
+
+export function getIssue(db: Database, id: string): Issue | null {
+  return db.prepare("SELECT * FROM issues WHERE id = $id").get({ $id: id }) as Issue | null
+}
+
+export function updateIssueStatus(db: Database, id: string, status: "open" | "resolved" | "ignored"): Issue | null {
+  return db.prepare("UPDATE issues SET status = $status WHERE id = $id RETURNING *")
+    .get({ $id: id, $status: status }) as Issue | null
+}

package/src/lib/page-auth.test.ts

@@ -0,0 +1,54 @@
+import { describe, expect, it } from "bun:test"
+import { createTestDb } from "../db/index.ts"
+import { deletePageAuth, getPageAuth, setPageAuth } from "./page-auth.ts"
+
+function seedPage(db: ReturnType<typeof createTestDb>) {
+  const p = db.prepare("INSERT INTO projects (name) VALUES ('app') RETURNING id").get() as { id: string }
+  const page = db.prepare("INSERT INTO pages (project_id, url) VALUES (?, 'https://app.com') RETURNING id").get(p.id) as { id: string }
+  return { projectId: p.id, pageId: page.id }
+}
+
+describe("page auth", () => {
+  it("sets and retrieves bearer auth", () => {
+    const db = createTestDb()
+    const { pageId } = seedPage(db)
+    setPageAuth(db, pageId, "bearer", "my-token-123")
+    const auth = getPageAuth(db, pageId)
+    expect(auth?.type).toBe("bearer")
+    expect(auth?.credentials).toBe("my-token-123")
+  })
+
+  it("credentials are encrypted at rest", () => {
+    const db = createTestDb()
+    const { pageId } = seedPage(db)
+    setPageAuth(db, pageId, "bearer", "secret-token")
+    const raw = db.prepare("SELECT credentials FROM page_auth WHERE page_id = ?").get(pageId) as { credentials: string }
+    // Raw value should NOT be the plaintext token
+    expect(raw.credentials).not.toBe("secret-token")
+    expect(raw.credentials).toContain(":")  // IV:encrypted format
+  })
+
+  it("upserts on duplicate page_id", () => {
+    const db = createTestDb()
+    const { pageId } = seedPage(db)
+    setPageAuth(db, pageId, "bearer", "token-v1")
+    setPageAuth(db, pageId, "bearer", "token-v2")
+    const auth = getPageAuth(db, pageId)
+    expect(auth?.credentials).toBe("token-v2")
+    const { c } = db.prepare("SELECT COUNT(*) as c FROM page_auth WHERE page_id = ?").get(pageId) as { c: number }
+    expect(c).toBe(1)
+  })
+
+  it("returns null for unknown page", () => {
+    const db = createTestDb()
+    expect(getPageAuth(db, "nope")).toBeNull()
+  })
+
+  it("deletes auth", () => {
+    const db = createTestDb()
+    const { pageId } = seedPage(db)
+    setPageAuth(db, pageId, "basic", "user:pass")
+    deletePageAuth(db, pageId)
+    expect(getPageAuth(db, pageId)).toBeNull()
+  })
+})

package/src/lib/page-auth.ts

@@ -0,0 +1,48 @@
+import type { Database } from "bun:sqlite"
+import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto"
+
+const SECRET_KEY = Buffer.from((process.env.LOGS_SECRET_KEY ?? "open-logs-default-key-32bytesXXX").padEnd(32).slice(0, 32))
+
+export interface PageAuth {
+  id: string
+  page_id: string
+  type: "cookie" | "bearer" | "basic"
+  credentials: string
+  created_at: string
+}
+
+function encrypt(text: string): string {
+  const iv = randomBytes(16)
+  const cipher = createCipheriv("aes-256-cbc", SECRET_KEY, iv)
+  const encrypted = Buffer.concat([cipher.update(text, "utf8"), cipher.final()])
+  return iv.toString("hex") + ":" + encrypted.toString("hex")
+}
+
+function decrypt(text: string): string {
+  const [ivHex, encHex] = text.split(":")
+  if (!ivHex || !encHex) return text
+  const iv = Buffer.from(ivHex, "hex")
+  const enc = Buffer.from(encHex, "hex")
+  const decipher = createDecipheriv("aes-256-cbc", SECRET_KEY, iv)
+  return Buffer.concat([decipher.update(enc), decipher.final()]).toString("utf8")
+}
+
+export function setPageAuth(db: Database, pageId: string, type: PageAuth["type"], credentials: string): PageAuth {
+  const encrypted = encrypt(credentials)
+  return db.prepare(`
+    INSERT INTO page_auth (page_id, type, credentials)
+    VALUES ($page_id, $type, $credentials)
+    ON CONFLICT(page_id) DO UPDATE SET type = excluded.type, credentials = excluded.credentials
+    RETURNING *
+  `).get({ $page_id: pageId, $type: type, $credentials: encrypted }) as PageAuth
+}
+
+export function getPageAuth(db: Database, pageId: string): { type: PageAuth["type"]; credentials: string } | null {
+  const row = db.prepare("SELECT * FROM page_auth WHERE page_id = $id").get({ $id: pageId }) as PageAuth | null
+  if (!row) return null
+  return { type: row.type, credentials: decrypt(row.credentials) }
+}
+
+export function deletePageAuth(db: Database, pageId: string): void {
+  db.run("DELETE FROM page_auth WHERE page_id = $id", { $id: pageId })
+}

package/src/lib/retention.test.ts

@@ -0,0 +1,42 @@
+import { describe, expect, it } from "bun:test"
+import { createTestDb } from "../db/index.ts"
+import { ingestBatch } from "./ingest.ts"
+import { runRetentionForProject, setRetentionPolicy } from "./retention.ts"
+
+function seedProject(db: ReturnType<typeof createTestDb>, name = "app") {
+  return db.prepare("INSERT INTO projects (name) VALUES (?) RETURNING id").get(name) as { id: string }
+}
+
+describe("retention", () => {
+  it("does nothing when under max_rows", () => {
+    const db = createTestDb()
+    const p = seedProject(db)
+    ingestBatch(db, Array.from({ length: 5 }, () => ({ level: "info" as const, message: "x", project_id: p.id })))
+    const result = runRetentionForProject(db, p.id)
+    expect(result.deleted).toBe(0)
+  })
+
+  it("enforces max_rows", () => {
+    const db = createTestDb()
+    const p = seedProject(db)
+    setRetentionPolicy(db, p.id, { max_rows: 3 })
+    ingestBatch(db, Array.from({ length: 10 }, () => ({ level: "info" as const, message: "x", project_id: p.id })))
+    runRetentionForProject(db, p.id)
+    const count = (db.prepare("SELECT COUNT(*) as c FROM logs WHERE project_id = ?").get(p.id) as { c: number }).c
+    expect(count).toBeLessThanOrEqual(3)
+  })
+
+  it("returns 0 for unknown project", () => {
+    const db = createTestDb()
+    expect(runRetentionForProject(db, "nope").deleted).toBe(0)
+  })
+
+  it("setRetentionPolicy updates project config", () => {
+    const db = createTestDb()
+    const p = seedProject(db)
+    setRetentionPolicy(db, p.id, { max_rows: 500, debug_ttl_hours: 1 })
+    const proj = db.prepare("SELECT max_rows, debug_ttl_hours FROM projects WHERE id = ?").get(p.id) as { max_rows: number; debug_ttl_hours: number }
+    expect(proj.max_rows).toBe(500)
+    expect(proj.debug_ttl_hours).toBe(1)
+  })
+})

package/src/lib/retention.ts

@@ -0,0 +1,62 @@
+import type { Database } from "bun:sqlite"
+
+interface RetentionConfig {
+  max_rows: number
+  debug_ttl_hours: number
+  info_ttl_hours: number
+  warn_ttl_hours: number
+  error_ttl_hours: number
+}
+
+const TTL_BY_LEVEL: Record<string, keyof RetentionConfig> = {
+  debug: "debug_ttl_hours",
+  info: "info_ttl_hours",
+  warn: "warn_ttl_hours",
+  error: "error_ttl_hours",
+  fatal: "error_ttl_hours",
+}
+
+export function runRetentionForProject(db: Database, projectId: string): { deleted: number } {
+  const project = db.prepare("SELECT * FROM projects WHERE id = $id").get({ $id: projectId }) as (RetentionConfig & { id: string }) | null
+  if (!project) return { deleted: 0 }
+
+  let deleted = 0
+
+  // TTL enforcement per level
+  for (const [level, configKey] of Object.entries(TTL_BY_LEVEL)) {
+    const ttlHours = project[configKey] as number
+    const cutoff = new Date(Date.now() - ttlHours * 3600 * 1000).toISOString()
+    const before = (db.prepare("SELECT COUNT(*) as c FROM logs WHERE project_id = $p AND level = $level AND timestamp < $cutoff").get({ $p: projectId, $level: level, $cutoff: cutoff }) as { c: number }).c
+    if (before > 0) {
+      db.prepare("DELETE FROM logs WHERE project_id = $p AND level = $level AND timestamp < $cutoff").run({ $p: projectId, $level: level, $cutoff: cutoff })
+      deleted += before
+    }
+  }
+
+  // max_rows enforcement
+  const total = (db.prepare("SELECT COUNT(*) as c FROM logs WHERE project_id = $p").get({ $p: projectId }) as { c: number }).c
+  if (total > project.max_rows) {
+    const toDelete = total - project.max_rows
+    db.prepare(`DELETE FROM logs WHERE id IN (SELECT id FROM logs WHERE project_id = $p ORDER BY timestamp ASC LIMIT ${toDelete})`).run({ $p: projectId })
+    deleted += toDelete
+  }
+
+  return { deleted }
+}
+
+export function runRetentionAll(db: Database): { deleted: number; projects: number } {
+  const projects = db.prepare("SELECT id FROM projects").all() as { id: string }[]
+  let deleted = 0
+  for (const p of projects) {
+    deleted += runRetentionForProject(db, p.id).deleted
+  }
+  return { deleted, projects: projects.length }
+}
+
+export function setRetentionPolicy(db: Database, projectId: string, config: Partial<RetentionConfig>): void {
+  const fields = Object.keys(config).map(k => `${k} = $${k}`).join(", ")
+  if (!fields) return
+  const params = Object.fromEntries(Object.entries(config).map(([k, v]) => [`$${k}`, v]))
+  params.$id = projectId
+  db.prepare(`UPDATE projects SET ${fields} WHERE id = $id`).run(params)
+}

package/src/lib/scanner.ts

@@ -1,5 +1,6 @@
 import type { Database } from "bun:sqlite"
 import { ingestBatch } from "./ingest.ts"
+import { getPageAuth } from "./page-auth.ts"
 import { saveSnapshot } from "./perf.ts"
 import { getPage, touchPage } from "./projects.ts"
 import type { LogEntry } from "../types/index.ts"
@@ -17,9 +18,27 @@ export async function scanPage(db: Database, projectId: string, pageId: string,
 
   const { chromium } = await import("playwright")
   const browser = await chromium.launch({ headless: true })
-  const context = await browser.newContext({
+
+  // Apply page auth if configured
+  const auth = getPageAuth(db, pageId)
+  const contextOptions: Parameters<typeof browser.newContext>[0] = {
     userAgent: "Mozilla/5.0 (@hasna/logs scanner) AppleWebKit/537.36",
-  })
+  }
+  if (auth?.type === "cookie") {
+    try { contextOptions.storageState = JSON.parse(auth.credentials) } catch { /* invalid */ }
+  } else if (auth?.type === "basic") {
+    const [username, password] = auth.credentials.split(":")
+    contextOptions.httpCredentials = { username: username ?? "", password: password ?? "" }
+  }
+
+  const context = await browser.newContext(contextOptions)
+
+  if (auth?.type === "bearer") {
+    await context.route("**/*", (route) => {
+      route.continue({ headers: { ...route.request().headers(), Authorization: `Bearer ${auth.credentials}` } })
+    })
+  }
+
   const browserPage = await context.newPage()
 
   const collected: LogEntry[] = []

package/src/lib/scheduler.ts

@@ -2,6 +2,7 @@ import type { Database } from "bun:sqlite"
 import cron from "node-cron"
 import { finishScanRun, createScanRun, listJobs, updateJob } from "./jobs.ts"
 import { listPages } from "./projects.ts"
+import { runRetentionAll } from "./retention.ts"
 import { scanPage } from "./scanner.ts"
 
 const tasks = new Map<string, cron.ScheduledTask>()
@@ -11,6 +12,11 @@ export function startScheduler(db: Database): void {
   for (const job of jobs) {
     scheduleJob(db, job.id, job.schedule, job.project_id, job.page_id ?? undefined)
   }
+  // Hourly retention runner
+  cron.schedule("0 * * * *", () => {
+    const result = runRetentionAll(db)
+    if (result.deleted > 0) console.log(`Retention: deleted ${result.deleted} logs across ${result.projects} project(s)`)
+  })
   console.log(`Scheduler started: ${tasks.size} job(s) active`)
 }