@vikasitai/vikasit-code 2.0.3 → 2.0.5

package/src/context/contextOptimizer.ts

@@ -0,0 +1,277 @@
+#!/usr/bin/env node
+
+/**
+ * Context Optimization System
+ *
+ * Reduces token consumption by:
+ * 1. Stripping unnecessary metadata
+ * 2. Smarter file previews
+ * 3. Condensed conversation summaries
+ * 4. Lazy context loading
+ */
+
+import { readFile } from "fs/promises"
+import path from "path"
+
+// Configuration
+const FILE_READ_LIMIT = 25000 // tokens
+const PREVIEW_LINES = 50 // lines of code to show
+const METADATA_LIMIT = 200 // chars to keep from file names
+const CONVERSATION_MAX = 10 // keep last 10 user-turns
+
+/**
+ * Strip tool-use metadata from messages before sending
+ * Saves ~100-500 tokens per operation
+ */
+export function stripToolUseMetadata(messages: any[]): any[] {
+  return messages.map((message) => {
+    if (!Array.isArray(message.content)) return message
+
+    return {
+      ...message,
+      content: message.content.map((block: any) => {
+        // Remove 'caller' from tool_use blocks
+        if (block.type === "tool_use") {
+          return {
+            type: "tool_use",
+            id: block.id,
+            name: block.name,
+            input: block.input,
+          }
+        }
+
+        // Strip tool_reference from tool_result
+        if (block.type === "tool_result") {
+          if (block.content && Array.isArray(block.content)) {
+            return {
+              ...block,
+              content: block.content.filter((c: any) => !isToolReference(c)),
+            }
+          }
+        }
+
+        return block
+      }),
+    }
+  })
+}
+
+/**
+ * Check if block is a tool_reference (Claude Code specific)
+ */
+function isToolReference(block: any): boolean {
+  return block.type === "tool_reference"
+}
+
+/**
+ * Truncate file content to save tokens
+ * Algorithms:
+ * 1. Always keep complete lines
+ * 2. If exceeds limit, keep first N lines + last N lines
+ * 3. Add truncation indicator
+ */
+export function truncateFileContent(
+  content: string,
+  maxTokens: number = FILE_READ_LIMIT,
+): { original: string; truncated: string; wasTruncated: boolean } {
+  // Estimate token count (rough: 4 chars = 1 token for code)
+  const tokenEstimate = content.length / 4
+  const lines = content.split("\n")
+
+  if (tokenEstimate <= maxTokens) {
+    return { original: content, truncated: content, wasTruncated: false }
+  }
+
+  const linesPerToken = 2 // average lines per token for code
+  const maxLines = Math.floor(maxTokens / linesPerToken)
+
+  // Split lines: header + footer
+  const mid = Math.floor(lines.length / 2)
+  const header = lines.slice(0, Math.min(lines.length, mid, maxLines)).join("\n")
+  const footer = lines.slice(Math.max(0, mid), lines.length).join("\n")
+
+  const truncated = truncateToMaxTokens(header, maxTokens) || footer
+
+  return {
+    original: content,
+    truncated,
+    wasTruncated: true,
+  }
+}
+
+/**
+ * Truncate markdown while preserving readability
+ * Removes nested code blocks, keeps paths
+ */
+export function truncateMarkdown(content: string, maxTokens: number): string {
+  const tokenEstimate = content.length / 4
+  if (tokenEstimate <= maxTokens) return content
+
+  // Replace long code blocks with paths
+  const lines = content.split("\n")
+  let result = []
+  let inCodeBlock = false
+  let braceCount = 0
+  let codeTokenCount = 0
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i]
+
+    if (inCodeBlock) {
+      codeTokenCount += line.length / 4
+      if (line.match(/^```/)) {
+        inCodeBlock = false
+        braceCount = 0
+        result.push(`  ...<${line.slice(1).trim()} truncated>...\n`)
+      }
+    } else if (line.includes("```")) {
+      inCodeBlock = true
+      result.push(`  <${line.slice(1).trim()} truncated>...\n`)
+    } else {
+      if (codeTokenCount > 0) {
+        codeTokenCount -= 10
+        result.push("  (package.json line) // truncated")
+      } else {
+        result.push(line)
+      }
+    }
+
+    if (codeTokenCount + result.length <= maxTokens) {
+      codeTokenCount += line.length / 4
+    }
+  }
+
+  return result.join("")
+}
+
+/**
+ * Truncate to max tokens, preserving complete lines
+ */
+function truncateToMaxTokens(content: string, maxTokens: number): string | null {
+  if (content.length / 4 <= maxTokens) return content
+
+  const lines = content.split("\n")
+  let result = []
+  let tokens = 0
+
+  for (const line of lines) {
+    const estimate = line.length / 4
+    if (tokens + estimate > maxTokens) {
+      return result.join("\n")
+    }
+    result.push(line)
+    tokens += estimate
+  }
+
+  return result.join("\n")
+}
+
+/**
+ * Extract just file paths from conversations
+ * Removes entire code blocks, function definitions
+ */
+export function extractFilePaths(content: string): { paths: string[]; totalTokens: number } {
+  const paths = []
+  let remainingTokens = 1000
+
+  // Find all path mentions
+  const pathRegex = /(?:created|modified) ([^\s\'"]+)/g
+  let match: RegExpExecArray | null
+
+  while ((match = pathRegex.exec(content)) !== null) {
+    const fileInfo = match[1]
+    if (fileInfo && fileInfo.length < METADATA_LIMIT) {
+      paths.push(fileInfo)
+      remainingTokens -= 10
+    }
+  }
+
+  return { paths, totalTokens: paths.length }
+}
+
+/**
+ * Compress repeated/identical messages
+ * Detects "I'm done" repeated patterns
+ */
+function compressRepeatedMessages(messages: any[]): any[] {
+  const compressed = []
+  let lastMessage = null
+
+  for (const message of messages) {
+    // Simple check for repeated content
+    const currentKey = JSON.stringify({
+      key: message.key,
+      type: message.type,
+    })
+
+    if (JSON.stringify(lastMessage) === currentKey) {
+      // Keep only first 3 of same messages
+      compressed.push(message)
+      if (compressed.length > 3) {
+        compressed.pop()
+      }
+      continue
+    }
+
+    compressed.push(message)
+    lastMessage = message
+  }
+
+  return compressed
+}
+
+/**
+ * Generate summary of conversation
+ * Saves tokens by summarizing instead of storing
+ */
+export function generateConversationSummary(messages: any[], maxTokens: number = 3000): string {
+  // Analyze conversation patterns
+  let summaryParts = [
+    `### Conversation Summary\n`,
+    `Turns: ${messages.length}\n`,
+    `Last message: ${messages[messages.length - 1].message},\n`,
+    `Key tasks: ${messages.filter((m: any) => m.message.includes("task") || m.message.includes("complete")).length}`,
+  ]
+
+  // Estimate token count
+  const result = summaryParts.join("")
+  if (result.length / 4 > maxTokens) {
+    // Take just the essentials
+    return `${summaryParts[0]}\n${summaryParts[1]}\n${summaryParts[2]}\nTotal: ${messages.length} turns`
+  }
+
+  return result
+}
+
+// CLI main
+if (import.meta.main) {
+  console.log("=== Context Optimization Demo ===\n")
+
+  const {
+    stripToolUseMetadata,
+    truncateFileContent,
+    truncateMarkdown,
+    compressRepeatedMessages,
+  } = require("./contextOptimizer.js")
+
+  // Example demonstrating all features
+  console.log("1. Token Reduction Techniques:\n")
+
+  console.log("2. File Truncation:")
+  const exampleFile = `// Important header comment file
+// More context information here
+// More metadata
+
+function doSomething() {
+  // ... 500 lines of code ...
+}
+
+// More code blocks
+// More code blocks
+
+// Final footer comment`
+  const results = truncateFileContent(exampleFile, FILE_READ_LIMIT)
+  console.log(`   Original: ${results.original.length} chars`)
+  console.log(`   Truncated: ${results.truncated.length} chars`)
+  console.log(`   Saved: ${((1 - results.truncated.length / results.original.length) * 100).toFixed(1)}%`)
+}

package/src/control-plane/adaptors/index.ts

@@ -0,0 +1,20 @@
+import { lazy } from "@/util/lazy"
+import type { Adaptor } from "../types"
+
+const ADAPTORS: Record<string, () => Promise<Adaptor>> = {
+  worktree: lazy(async () => (await import("./worktree")).WorktreeAdaptor),
+}
+
+export function getAdaptor(type: string): Promise<Adaptor> {
+  return ADAPTORS[type]()
+}
+
+export function installAdaptor(type: string, adaptor: Adaptor) {
+  // This is experimental: mostly used for testing right now, but we
+  // will likely allow this in the future. Need to figure out the
+  // TypeScript story
+
+  // @ts-expect-error we force the builtin types right now, but we
+  // will implement a way to extend the types for custom adaptors
+  ADAPTORS[type] = () => adaptor
+}

package/src/control-plane/adaptors/worktree.ts

@@ -0,0 +1,38 @@
+import z from "zod"
+import { Worktree } from "@/worktree"
+import { type Adaptor, WorkspaceInfo } from "../types"
+
+const Config = WorkspaceInfo.extend({
+  name: WorkspaceInfo.shape.name.unwrap(),
+  branch: WorkspaceInfo.shape.branch.unwrap(),
+  directory: WorkspaceInfo.shape.directory.unwrap(),
+})
+
+type Config = z.infer<typeof Config>
+
+export const WorktreeAdaptor: Adaptor = {
+  async configure(info) {
+    const worktree = await Worktree.makeWorktreeInfo(info.name ?? undefined)
+    return {
+      ...info,
+      name: worktree.name,
+      branch: worktree.branch,
+      directory: worktree.directory,
+    }
+  },
+  async create(info) {
+    const config = Config.parse(info)
+    await Worktree.createFromInfo({
+      name: config.name,
+      directory: config.directory,
+      branch: config.branch,
+    })
+  },
+  async remove(info) {
+    const config = Config.parse(info)
+    await Worktree.remove({ directory: config.directory })
+  },
+  async fetch(_info, _input: RequestInfo | URL, _init?: RequestInit) {
+    throw new Error("fetch not implemented")
+  },
+}

package/src/control-plane/schema.ts

@@ -0,0 +1,17 @@
+import { Schema } from "effect"
+import z from "zod"
+
+import { withStatics } from "@/util/schema"
+import { Identifier } from "@/id/id"
+
+const workspaceIdSchema = Schema.String.pipe(Schema.brand("WorkspaceID"))
+
+export type WorkspaceID = typeof workspaceIdSchema.Type
+
+export const WorkspaceID = workspaceIdSchema.pipe(
+  withStatics((schema: typeof workspaceIdSchema) => ({
+    make: (id: string) => schema.makeUnsafe(id),
+    ascending: (id?: string) => schema.makeUnsafe(Identifier.ascending("workspace", id)),
+    zod: Identifier.schema("workspace").pipe(z.custom<WorkspaceID>()),
+  })),
+)

package/src/control-plane/sse.ts

@@ -0,0 +1,66 @@
+export async function parseSSE(
+  body: ReadableStream<Uint8Array>,
+  signal: AbortSignal,
+  onEvent: (event: unknown) => void,
+) {
+  const reader = body.getReader()
+  const decoder = new TextDecoder()
+  let buf = ""
+  let last = ""
+  let retry = 1000
+
+  const abort = () => {
+    void reader.cancel().catch(() => undefined)
+  }
+
+  signal.addEventListener("abort", abort)
+
+  try {
+    while (!signal.aborted) {
+      const chunk = await reader.read().catch(() => ({ done: true, value: undefined as Uint8Array | undefined }))
+      if (chunk.done) break
+
+      buf += decoder.decode(chunk.value, { stream: true })
+      buf = buf.replace(/\r\n/g, "\n").replace(/\r/g, "\n")
+
+      const chunks = buf.split("\n\n")
+      buf = chunks.pop() ?? ""
+
+      chunks.forEach((chunk) => {
+        const data: string[] = []
+        chunk.split("\n").forEach((line) => {
+          if (line.startsWith("data:")) {
+            data.push(line.replace(/^data:\s*/, ""))
+            return
+          }
+          if (line.startsWith("id:")) {
+            last = line.replace(/^id:\s*/, "")
+            return
+          }
+          if (line.startsWith("retry:")) {
+            const parsed = Number.parseInt(line.replace(/^retry:\s*/, ""), 10)
+            if (!Number.isNaN(parsed)) retry = parsed
+          }
+        })
+
+        if (!data.length) return
+        const raw = data.join("\n")
+        try {
+          onEvent(JSON.parse(raw))
+        } catch {
+          onEvent({
+            type: "sse.message",
+            properties: {
+              data: raw,
+              id: last || undefined,
+              retry,
+            },
+          })
+        }
+      })
+    }
+  } finally {
+    signal.removeEventListener("abort", abort)
+    reader.releaseLock()
+  }
+}

package/src/control-plane/types.ts

@@ -0,0 +1,21 @@
+import z from "zod"
+import { ProjectID } from "@/project/schema"
+import { WorkspaceID } from "./schema"
+
+export const WorkspaceInfo = z.object({
+  id: WorkspaceID.zod,
+  type: z.string(),
+  branch: z.string().nullable(),
+  name: z.string().nullable(),
+  directory: z.string().nullable(),
+  extra: z.unknown().nullable(),
+  projectID: ProjectID.zod,
+})
+export type WorkspaceInfo = z.infer<typeof WorkspaceInfo>
+
+export type Adaptor = {
+  configure(input: WorkspaceInfo): WorkspaceInfo | Promise<WorkspaceInfo>
+  create(input: WorkspaceInfo, from?: WorkspaceInfo): Promise<void>
+  remove(config: WorkspaceInfo): Promise<void>
+  fetch(config: WorkspaceInfo, input: RequestInfo | URL, init?: RequestInit): Promise<Response>
+}

package/src/control-plane/workspace.sql.ts

@@ -0,0 +1,17 @@
+import { sqliteTable, text } from "drizzle-orm/sqlite-core"
+import { ProjectTable } from "../project/project.sql"
+import type { ProjectID } from "../project/schema"
+import type { WorkspaceID } from "./schema"
+
+export const WorkspaceTable = sqliteTable("workspace", {
+  id: text().$type<WorkspaceID>().primaryKey(),
+  type: text().notNull(),
+  branch: text(),
+  name: text(),
+  directory: text(),
+  extra: text({ mode: "json" }),
+  project_id: text()
+    .$type<ProjectID>()
+    .notNull()
+    .references(() => ProjectTable.id, { onDelete: "cascade" }),
+})

package/src/control-plane/workspace.ts

@@ -0,0 +1,154 @@
+import z from "zod"
+import { setTimeout as sleep } from "node:timers/promises"
+import { fn } from "@/util/fn"
+import { Database, eq } from "@/storage/db"
+import { Project } from "@/project/project"
+import { BusEvent } from "@/bus/bus-event"
+import { GlobalBus } from "@/bus/global"
+import { Log } from "@/util/log"
+import { ProjectID } from "@/project/schema"
+import { WorkspaceTable } from "./workspace.sql"
+import { getAdaptor } from "./adaptors"
+import { WorkspaceInfo } from "./types"
+import { WorkspaceID } from "./schema"
+import { parseSSE } from "./sse"
+
+export namespace Workspace {
+  export const Event = {
+    Ready: BusEvent.define(
+      "workspace.ready",
+      z.object({
+        name: z.string(),
+      }),
+    ),
+    Failed: BusEvent.define(
+      "workspace.failed",
+      z.object({
+        message: z.string(),
+      }),
+    ),
+  }
+
+  export const Info = WorkspaceInfo.meta({
+    ref: "Workspace",
+  })
+  export type Info = z.infer<typeof Info>
+
+  function fromRow(row: typeof WorkspaceTable.$inferSelect): Info {
+    return {
+      id: row.id,
+      type: row.type,
+      branch: row.branch,
+      name: row.name,
+      directory: row.directory,
+      extra: row.extra,
+      projectID: row.project_id,
+    }
+  }
+
+  const CreateInput = z.object({
+    id: WorkspaceID.zod.optional(),
+    type: Info.shape.type,
+    branch: Info.shape.branch,
+    projectID: ProjectID.zod,
+    extra: Info.shape.extra,
+  })
+
+  export const create = fn(CreateInput, async (input) => {
+    const id = WorkspaceID.ascending(input.id)
+    const adaptor = await getAdaptor(input.type)
+
+    const config = await adaptor.configure({ ...input, id, name: null, directory: null })
+
+    const info: Info = {
+      id,
+      type: config.type,
+      branch: config.branch ?? null,
+      name: config.name ?? null,
+      directory: config.directory ?? null,
+      extra: config.extra ?? null,
+      projectID: input.projectID,
+    }
+
+    Database.use((db) => {
+      db.insert(WorkspaceTable)
+        .values({
+          id: info.id,
+          type: info.type,
+          branch: info.branch,
+          name: info.name,
+          directory: info.directory,
+          extra: info.extra,
+          project_id: info.projectID,
+        })
+        .run()
+    })
+
+    await adaptor.create(config)
+    return info
+  })
+
+  export function list(project: Project.Info) {
+    const rows = Database.use((db) =>
+      db.select().from(WorkspaceTable).where(eq(WorkspaceTable.project_id, project.id)).all(),
+    )
+    return rows.map(fromRow).sort((a, b) => a.id.localeCompare(b.id))
+  }
+
+  export const get = fn(WorkspaceID.zod, async (id) => {
+    const row = Database.use((db) => db.select().from(WorkspaceTable).where(eq(WorkspaceTable.id, id)).get())
+    if (!row) return
+    return fromRow(row)
+  })
+
+  export const remove = fn(WorkspaceID.zod, async (id) => {
+    const row = Database.use((db) => db.select().from(WorkspaceTable).where(eq(WorkspaceTable.id, id)).get())
+    if (row) {
+      const info = fromRow(row)
+      const adaptor = await getAdaptor(row.type)
+      adaptor.remove(info)
+      Database.use((db) => db.delete(WorkspaceTable).where(eq(WorkspaceTable.id, id)).run())
+      return info
+    }
+  })
+  const log = Log.create({ service: "workspace-sync" })
+
+  async function workspaceEventLoop(space: Info, stop: AbortSignal) {
+    while (!stop.aborted) {
+      const adaptor = await getAdaptor(space.type)
+      const res = await adaptor.fetch(space, "/event", { method: "GET", signal: stop }).catch(() => undefined)
+      if (!res || !res.ok || !res.body) {
+        await sleep(1000)
+        continue
+      }
+      await parseSSE(res.body, stop, (event) => {
+        GlobalBus.emit("event", {
+          directory: space.id,
+          payload: event,
+        })
+      })
+      // Wait 250ms and retry if SSE connection fails
+      await sleep(250)
+    }
+  }
+
+  export function startSyncing(project: Project.Info) {
+    const stop = new AbortController()
+    const spaces = list(project).filter((space) => space.type !== "worktree")
+
+    spaces.forEach((space) => {
+      void workspaceEventLoop(space, stop.signal).catch((error) => {
+        log.warn("workspace sync listener failed", {
+          workspaceID: space.id,
+          error,
+        })
+      })
+    })
+
+    return {
+      async stop() {
+        stop.abort()
+      },
+    }
+  }
+}