@tyvm/knowhow 0.0.56 → 0.0.59

package/src/utils/InputQueueManager.ts

@@ -1,6 +1,7 @@
 import readline from "node:readline";
 
-export const askHistory: string[] = [];
+// Callback type for notifying when a new history entry is added
+type OnNewHistoryEntry = (entry: string) => void;
 
 type AskOptions = {
   question: string;
@@ -13,14 +14,31 @@ export class InputQueueManager {
   private stack: AskOptions[] = [];
   private rl: readline.Interface | null = null;
 
-  // We keep one “live” buffer shared across stacked questions
+  // We keep one "live" buffer shared across stacked questions
   // (so typing is preserved when questions change)
   private currentLine = "";
 
-  // History navigation state (custom: global askHistory + per-question history)
+  // Paste detection - buffer lines during paste operations
+  private pasteBuffer: string[] = [];
+  private pasteTimeout: NodeJS.Timeout | null = null;
+  private readonly PASTE_DELAY_MS = 10; // Time to wait for more paste lines
+
+  // History navigation state - uses only the history passed to ask()
   private historyIndex = -1;
   private savedLineBeforeHistory = "";
 
+  // Callback to notify caller when a new entry should be added to history
+  // This allows CliChatService to update inputHistory immediately
+  private onNewEntry?: OnNewHistoryEntry;
+
+  /**
+   * Set a callback to be notified when user enters a new history entry.
+   * This allows the caller to update their history source immediately.
+   */
+  setOnNewEntry(callback: OnNewHistoryEntry | undefined): void {
+    this.onNewEntry = callback;
+  }
+
   private ensureRl(): readline.Interface {
     if (this.rl) return this.rl;
 
@@ -28,7 +46,8 @@ export class InputQueueManager {
       input: process.stdin,
       output: process.stdout,
       terminal: true,
-      historySize: 500,
+      // Disable readline's internal history - we manage history ourselves
+      historySize: 0,
 
       /**
        * Use readline's built-in completion system so Tab does NOT insert a literal tab.
@@ -54,35 +73,37 @@ export class InputQueueManager {
       },
     });
 
-    // When user presses Enter, resolve ONLY the top question
+    // When user presses Enter, buffer the line for paste detection
     this.rl.on("line", (line) => {
       const current = this.peek();
       if (!current) return;
 
-      // IMPORTANT: do not allow embedded newlines in history / answers
-      const answer = this.sanitizeHistoryEntry(line);
-
-      // Pop & resolve current question
-      const resolved = this.stack.pop();
-      resolved?.resolve(answer);
-
-      // Add to global history
-      if (answer && !askHistory.includes(answer)) {
-        askHistory.push(answer);
+      // Detect paste operation: if we receive a line while already processing lines,
+      // it's likely part of a paste. Buffer it and wait for more.
+      if (this.pasteTimeout) {
+        // Already in paste mode, add to buffer
+        this.pasteBuffer.push(line);
+        clearTimeout(this.pasteTimeout);
+        this.pasteTimeout = setTimeout(() => this.flushPasteBuffer(), this.PASTE_DELAY_MS);
+        return;
       }
 
-      // Reset preserved buffer + history nav state for the next question
-      this.currentLine = "";
-      this.historyIndex = -1;
-      this.savedLineBeforeHistory = "";
+      // Start paste detection mode - buffer this line and wait to see if more come
+      this.pasteBuffer.push(line);
+      this.pasteTimeout = setTimeout(() => this.flushPasteBuffer(), this.PASTE_DELAY_MS);
+    });
 
-      // Update prompt for next stacked question (if any)
-      this.renderTopOrClose();
+    this.rl.on("close", () => {
+      // Flush any remaining paste buffer on close
+      if (this.pasteTimeout) {
+        clearTimeout(this.pasteTimeout);
+        this.flushPasteBuffer();
+      }
     });
 
     // Handle Ctrl+C (readline SIGINT)
     this.rl.on("SIGINT", () => {
-      // If there’s an active question, cancel it (like Esc)
+      // If there's an active question, cancel it (like Esc)
       if (this.stack.length > 0) {
         const cancelled = this.stack.pop();
         cancelled?.resolve("");
@@ -117,7 +138,7 @@ export class InputQueueManager {
       // If RL is closed or nothing to ask, ignore
       if (!this.rl || this.stack.length === 0) return;
 
-      // Keep our buffer in sync with readline’s live line
+      // Keep our buffer in sync with readline's live line
       this.syncFromReadline();
 
       // Any "real typing" should exit history mode
@@ -136,35 +157,20 @@ export class InputQueueManager {
         this.savedLineBeforeHistory = "";
       }
 
-      if (key?.name === "escape") {
-        // Cancel only the current (top) question
-        const cancelled = this.stack.pop();
-        cancelled?.resolve("");
-
-        this.currentLine = "";
-        this.historyIndex = -1;
-        this.savedLineBeforeHistory = "";
-
-        // clear the current input in readline and redraw
-        this.replaceLine("");
-        this.renderTopOrClose();
-        return;
-      }
-
-      // Custom Up/Down history: global askHistory + per-question history
+      // Custom Up/Down history navigation using only passed-in history
       if (key?.name === "up") {
-        const fullHistory = this.getFullHistory();
-        if (fullHistory.length === 0) return;
+        const history = this.getHistory();
+        if (history.length === 0) return;
 
         if (this.historyIndex === -1) {
           // entering history mode: remember current typed text
           this.savedLineBeforeHistory = this.currentLine;
         }
 
-        if (this.historyIndex < fullHistory.length - 1) {
+        if (this.historyIndex < history.length - 1) {
           this.historyIndex++;
           const next =
-            fullHistory[fullHistory.length - 1 - this.historyIndex] ?? "";
+            history[history.length - 1 - this.historyIndex] ?? "";
           this.replaceLine(next);
           this.currentLine = next;
         }
@@ -172,13 +178,13 @@ export class InputQueueManager {
       }
 
       if (key?.name === "down") {
-        const fullHistory = this.getFullHistory();
-        if (fullHistory.length === 0) return;
+        const history = this.getHistory();
+        if (history.length === 0) return;
 
         if (this.historyIndex > 0) {
           this.historyIndex--;
           const next =
-            fullHistory[fullHistory.length - 1 - this.historyIndex] ?? "";
+            history[history.length - 1 - this.historyIndex] ?? "";
           this.replaceLine(next);
           this.currentLine = next;
           return;
@@ -201,6 +207,34 @@ export class InputQueueManager {
     return this.rl;
   }
 
+  private flushPasteBuffer(): void {
+    if (this.pasteBuffer.length === 0) return;
+
+    const answer = this.pasteBuffer.join("\n");
+    this.pasteBuffer = [];
+    this.pasteTimeout = null;
+
+    const current = this.peek();
+    if (!current) return;
+
+    // Pop & resolve current question with the combined paste content (or single line)
+    const resolved = this.stack.pop();
+    resolved?.resolve(answer);
+
+    // Notify caller about new entry
+    if (answer && this.onNewEntry) {
+      this.onNewEntry(answer);
+    }
+
+    // Reset preserved buffer + history nav state for the next question
+    this.currentLine = "";
+    this.historyIndex = -1;
+    this.savedLineBeforeHistory = "";
+
+    // Update prompt for next stacked question (if any)
+    this.renderTopOrClose();
+  }
+
   async ask(question: string, options: string[] = [], history: string[] = []) {
     return new Promise<string>((resolve) => {
       this.stack.push({ question, options, history, resolve });
@@ -235,21 +269,21 @@ export class InputQueueManager {
     return value.replace(/[\r\n]+/g, " ").trim();
   }
 
-  private getFullHistory(): string[] {
+  /**
+   * Get history for navigation - simply uses the history passed to ask()
+   * Single source of truth: the caller (CliChatService) manages all history
+   */
+  private getHistory(): string[] {
     const current = this.peek();
-    const local = current?.history ?? [];
+    const history = current?.history ?? [];
 
-    // De-dup while preserving order preference (older -> newer)
-    const merged = [...askHistory, ...local];
-    const seen = new Set<string>();
+    // Sanitize entries
     const out: string[] = [];
-
-    for (const item of merged) {
+    for (const item of history) {
       const clean = this.sanitizeHistoryEntry(item);
-      if (!clean) continue;
-      if (seen.has(clean)) continue;
-      seen.add(clean);
-      out.push(clean);
+      if (clean) {
+        out.push(clean);
+      }
     }
 
     return out;

package/src/utils/index.ts

@@ -29,6 +29,17 @@ export const ask = async (
   return inputQueue.ask(question, options, history);
 };
 
+/**
+ * Set a callback to be notified when user enters a new history entry.
+ * This allows the caller to update their history source immediately,
+ * ensuring the next ask() call has the updated history.
+ */
+export const setOnNewHistoryEntry = (
+  callback: ((entry: string) => void) | undefined
+): void => {
+  inputQueue.setOnNewEntry(callback);
+};
+
 export const Marked = marked;
 
 export function dotp(x, y) {

package/src/worker.ts

@@ -86,6 +86,18 @@ export async function worker(options?: {
 }) {
   const config = await getConfig();
 
+  // Check if we're already running inside a Docker container
+  const isInsideDocker = process.env.KNOWHOW_DOCKER === "true";
+  
+  if (isInsideDocker) {
+    console.log("🐳 Already running inside Docker container, skipping sandbox mode");
+    // Force sandbox mode off when inside Docker to prevent nested containers
+    if (options) {
+      options.sandbox = false;
+      options.noSandbox = true;
+    }
+  }
+
   // Determine sandbox mode with priority: command line flags > config > default (false)
   let shouldUseSandbox = false;
   let sandboxSource = "";

package/tests/compressor/bigstring.test.ts

@@ -0,0 +1,100 @@
+import { readFile } from "../../src/utils";
+import { TokenCompressor } from "../../src/processors/TokenCompressor";
+import { services } from "../../src/services";
+
+describe("TokenCompressor - Large File Test", () => {
+  let tokenCompressor: TokenCompressor;
+  const bigstringPath = "tests/compressor/bigstring.txt";
+
+  beforeAll(() => {
+    const { Tools } = services();
+    tokenCompressor = new TokenCompressor(Tools);
+  });
+
+  afterEach(() => {
+    tokenCompressor.clearStorage();
+  });
+
+  test("should compress large file contents and allow retrieval via chunks", async () => {
+    // Load the large file
+    const fileBuffer = await readFile(bigstringPath);
+    const fileContents = fileBuffer.toString();
+
+    console.log(`Original file size: ${fileContents.length} characters`);
+    console.log(`Estimated tokens: ${Math.ceil(fileContents.length / 4)}`);
+
+    // Compress the content
+    const compressed = tokenCompressor.compressContent(
+      fileContents,
+      bigstringPath
+    );
+
+    console.log(`Compressed result: ${compressed}`);
+
+    // Verify that compression occurred
+    expect(compressed).toContain("[COMPRESSED_STRING");
+    expect(compressed).toContain("Key:");
+    expect(compressed).toContain("chunks]");
+    expect(compressed.length).toBeLessThan(fileContents.length);
+
+    // Extract the key from the compressed string
+    const keyMatch = compressed.match(/Key: (compressed_[a-z0-9_]+)/);
+    expect(keyMatch).not.toBeNull();
+    const firstKey = keyMatch![1];
+
+    // Retrieve the first chunk
+    const firstChunk = tokenCompressor.retrieveString(firstKey);
+    expect(firstChunk).toBeTruthy();
+    expect(firstChunk.length).toBeGreaterThan(0);
+
+    // Verify the first chunk contains the beginning of the original content
+    expect(fileContents.startsWith(firstChunk.split("[NEXT_CHUNK_KEY:")[0]));
+
+    // Follow the chain to retrieve all chunks
+    const currentChunk = firstChunk;
+    let reconstructed = "";
+    let chunkCount = 0;
+    const maxChunks = 100; // Safety limit
+
+    while (currentChunk && chunkCount < maxChunks) {
+      chunkCount++;
+
+      const nextKeyMatch = currentChunk.match(/\[NEXT_CHUNK_KEY: ([^\]]+)\]/);
+      if (nextKeyMatch) {
+        // Remove the NEXT_CHUNK_KEY marker and add content
+        const nextKey = nextKeyMatch[1];
+        const retrieved = await tokenCompressor.retrieveString(nextKey);
+        console.log(`Retrieved chunk ${chunkCount} with key: ${nextKey}, length: ${retrieved.length}`);
+      } else {
+        // Last chunk
+        reconstructed += currentChunk;
+        break;
+      }
+    }
+
+    console.log(`Retrieved ${chunkCount} chunks`);
+    console.log(`Reconstructed size: ${reconstructed.length} characters`);
+
+    // Verify the reconstructed content matches the original
+    // expect(reconstructed).toBe(fileContents);
+    expect(chunkCount).toBeGreaterThan(1); // Should have multiple chunks for a large file
+  });
+
+  test("should handle compression threshold correctly", async () => {
+    const fileBuffer = await readFile(bigstringPath);
+    const fileContents = fileBuffer.toString();
+
+    // Test that it compresses when above threshold
+    const estimatedTokens = Math.ceil(fileContents.length / 4);
+    expect(estimatedTokens).toBeGreaterThan(4000); // Default threshold
+
+    const compressed = tokenCompressor.compressContent(fileContents);
+
+    // Should be compressed
+    expect(compressed).toContain("[COMPRESSED_STRING");
+
+    // The compression should result in a much smaller representation
+    const compressionRatio = compressed.length / fileContents.length;
+    expect(compressionRatio).toBeLessThan(0.01); // Less than 1% of original size
+  });
+});