@codewithdan/zingit 0.0.1

package/package.json

@@ -0,0 +1,69 @@
+{
+  "name": "@codewithdan/zingit",
+  "version": "0.0.1",
+  "description": "AI-powered UI annotation tool - point, annotate, and let AI fix it",
+  "type": "module",
+  "engines": {
+    "node": ">=22.0.0"
+  },
+  "bin": {
+    "zingit": "./bin/cli.js"
+  },
+  "files": [
+    "bin",
+    "server/dist",
+    "client/dist",
+    "README.md",
+    "AGENTS.md"
+  ],
+  "main": "client/dist/zingit-client.js",
+  "scripts": {
+    "build": "npm run build:client && npm run build:server",
+    "build:client": "cd client && npm run build",
+    "build:server": "cd server && npm run build",
+    "dev": "concurrently \"npm run dev:server\" \"npm run dev:client\"",
+    "dev:server": "cd server && npm run dev",
+    "dev:client": "cd client && npm run dev",
+    "prepublishOnly": "npm run build",
+    "publish": "npm run build && npm publish --access public"
+  },
+  "keywords": [
+    "ui",
+    "annotation",
+    "ai",
+    "agent",
+    "claude",
+    "copilot",
+    "codex",
+    "development",
+    "tool"
+  ],
+  "author": "Dan Wahlin",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/danwahlin/zingit"
+  },
+  "bugs": {
+    "url": "https://github.com/danwahlin/zingit/issues"
+  },
+  "homepage": "https://github.com/danwahlin/zingit#readme",
+  "dependencies": {
+    "@anthropic-ai/claude-agent-sdk": "^0.2.17",
+    "@github/copilot-sdk": "^0.1.16",
+    "@openai/codex-sdk": "^0.89.0",
+    "diff": "^8.0.3",
+    "uuid": "^11.1.0",
+    "ws": "^8.19.0",
+    "zod": "^4.3.6"
+  },
+  "devDependencies": {
+    "@types/diff": "^6.0.0",
+    "@types/node": "^25.0.10",
+    "@types/uuid": "^10.0.0",
+    "@types/ws": "^8.18.1",
+    "concurrently": "^9.1.2",
+    "tsx": "^4.21.0",
+    "typescript": "^5.9.3"
+  }
+}

package/server/dist/agents/base.d.ts

@@ -0,0 +1,20 @@
+import type { WebSocket } from 'ws';
+import type { Agent, AgentSession, BatchData, ImageContent } from '../types.js';
+export declare abstract class BaseAgent implements Agent {
+    abstract name: string;
+    abstract model: string;
+    abstract start(): Promise<void>;
+    abstract stop(): Promise<void>;
+    abstract createSession(ws: WebSocket, projectDir: string): Promise<AgentSession>;
+    /**
+     * Format a prompt with image metadata for agents that don't support native multimodal.
+     * This adds text descriptions of the images to the prompt.
+     */
+    protected formatPromptWithImageMetadata(prompt: string, images?: ImageContent[]): string;
+    /**
+     * Extract images from batch data annotations
+     * Returns an array of ImageContent objects for annotations that have screenshots
+     */
+    extractImages(data: BatchData): ImageContent[];
+    formatPrompt(data: BatchData, projectDir: string): string;
+}

package/server/dist/agents/base.js

@@ -0,0 +1,136 @@
+// server/src/agents/base.ts
+// Maximum image size: 10MB (base64 encoded)
+const MAX_IMAGE_SIZE_BYTES = 10 * 1024 * 1024;
+export class BaseAgent {
+    /**
+     * Format a prompt with image metadata for agents that don't support native multimodal.
+     * This adds text descriptions of the images to the prompt.
+     */
+    formatPromptWithImageMetadata(prompt, images) {
+        if (!images || images.length === 0) {
+            return prompt;
+        }
+        let header = `The following screenshots are provided for visual context:\n\n`;
+        for (const img of images) {
+            const sizeKB = Math.round(img.base64.length / 1024);
+            header += `${img.label || 'Screenshot'}:\n[Image data: ${img.mediaType}, ${sizeKB}KB]\n\n`;
+        }
+        return `${header}---\n\n${prompt}`;
+    }
+    /**
+     * Extract images from batch data annotations
+     * Returns an array of ImageContent objects for annotations that have screenshots
+     */
+    extractImages(data) {
+        const images = [];
+        data.annotations.forEach((ann, i) => {
+            if (ann.screenshot) {
+                let base64Data = ann.screenshot;
+                let mediaType = 'image/png'; // Default
+                // Extract media type and base64 data from data URL if present
+                if (base64Data.startsWith('data:')) {
+                    const prefixMatch = base64Data.match(/^data:(image\/[a-z+]+);base64,/i);
+                    if (prefixMatch) {
+                        mediaType = prefixMatch[1];
+                        base64Data = base64Data.slice(prefixMatch[0].length);
+                    }
+                    else {
+                        // Fallback: just find the comma and extract data
+                        const commaIndex = base64Data.indexOf(',');
+                        if (commaIndex > 0) {
+                            base64Data = base64Data.slice(commaIndex + 1);
+                        }
+                    }
+                }
+                // Validate base64 format:
+                // 1. Check for valid characters
+                // 2. Check length is divisible by 4 (required for valid base64)
+                // 3. Check padding is correct
+                const base64Regex = /^[A-Za-z0-9+/]*={0,2}$/;
+                if (!base64Data || !base64Regex.test(base64Data) || base64Data.length % 4 !== 0) {
+                    console.warn(`ZingIt: Invalid base64 data in annotation ${i + 1}, skipping screenshot`);
+                    return; // Skip this annotation's screenshot
+                }
+                // Check image size limit (base64 is ~33% larger than binary)
+                const estimatedBinarySize = Math.ceil(base64Data.length * 0.75);
+                if (estimatedBinarySize > MAX_IMAGE_SIZE_BYTES) {
+                    console.warn(`ZingIt: Image in annotation ${i + 1} exceeds ${MAX_IMAGE_SIZE_BYTES / 1024 / 1024}MB limit, skipping`);
+                    return; // Skip oversized image
+                }
+                // Validate that base64 can be decoded (catches corrupted data)
+                try {
+                    Buffer.from(base64Data, 'base64');
+                }
+                catch (err) {
+                    console.warn(`ZingIt: Failed to decode base64 in annotation ${i + 1}, skipping screenshot:`, err);
+                    return; // Skip this annotation's screenshot
+                }
+                images.push({
+                    base64: base64Data,
+                    mediaType,
+                    label: `Screenshot of Annotation ${i + 1}: ${ann.identifier}`
+                });
+            }
+        });
+        return images;
+    }
+    formatPrompt(data, projectDir) {
+        let prompt = `You are fixing UI issues on a webpage. The project is located at: ${projectDir}
+
+Page: ${data.pageTitle}
+URL: ${data.pageUrl}
+
+`;
+        data.annotations.forEach((ann, i) => {
+            prompt += `---
+
+## Annotation ${i + 1}: ${ann.identifier}
+
+**Requested Change:** ${ann.notes}
+
+**Target Element HTML:**
+\`\`\`html
+${ann.html}
+\`\`\`
+
+${ann.siblingContext ? `**Position in DOM:**
+${ann.siblingContext}
+
+` : ''}${ann.parentHtml ? `**Parent Context (target marked with data-zingit-target="true"):**
+\`\`\`html
+${ann.parentHtml}
+\`\`\`
+
+` : ''}${ann.textContent ? `**Text Content:** "${ann.textContent}"` : ''}
+${ann.selectedText ? `**Selected Text:** "${ann.selectedText}"` : ''}
+${ann.parentContext ? `**Parent Path:** \`${ann.parentContext}\`` : ''}
+**CSS Selector:** \`${ann.selector}\`
+
+`;
+        });
+        // Check if any annotations have screenshots
+        const hasScreenshots = data.annotations.some(ann => ann.screenshot);
+        prompt += `
+CRITICAL INSTRUCTIONS:
+1. CAREFULLY identify the CORRECT element to modify:
+   - The "Position in DOM" shows which element among siblings is the target (marked with "← THIS ONE")
+   - The "Parent Context" HTML shows the element with data-zingit-target="true" attribute - THAT is the one to change
+     (Note: data-zingit-target is the marker attribute - keep in sync with client/src/services/selector.ts)
+   - Do NOT change other similar elements that happen to have matching text
+
+2. Search for the parent context HTML in source files to find the exact location
+
+3. Make ONLY the requested change to the specific marked element
+
+4. If there are multiple similar elements (e.g., multiple <button> tags), use the positional context to identify the correct one`;
+        if (hasScreenshots) {
+            prompt += `
+
+5. Screenshots have been provided showing the current visual state of the annotated elements. Use these images to:
+   - Better understand the visual context and styling of the elements
+   - Identify the exact appearance that needs to be changed
+   - Verify you're targeting the correct element based on its visual representation`;
+        }
+        return prompt;
+    }
+}

package/server/dist/agents/claude.d.ts

@@ -0,0 +1,18 @@
+import type { WebSocket } from 'ws';
+import { BaseAgent } from './base.js';
+import type { AgentSession } from '../types.js';
+export declare class ClaudeCodeAgent extends BaseAgent {
+    name: string;
+    model: string;
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    /**
+     * Build content blocks for multimodal message with images and text
+     */
+    private buildContentBlocks;
+    /**
+     * Create a generator that yields the initial user message with optional images
+     */
+    private createMessageGenerator;
+    createSession(ws: WebSocket, projectDir: string): Promise<AgentSession>;
+}

package/server/dist/agents/claude.js

@@ -0,0 +1,141 @@
+// server/src/agents/claude.ts
+// Agent that uses Claude Agent SDK
+import { query } from '@anthropic-ai/claude-agent-sdk';
+import { BaseAgent } from './base.js';
+export class ClaudeCodeAgent extends BaseAgent {
+    name = 'claude';
+    model = 'claude-sonnet-4-20250514';
+    async start() {
+        console.log(`✓ Claude Agent SDK initialized (model: ${this.model})`);
+    }
+    async stop() {
+        // SDK handles cleanup automatically
+    }
+    /**
+     * Build content blocks for multimodal message with images and text
+     */
+    buildContentBlocks(prompt, images) {
+        const content = [];
+        // Add images first so Claude sees them before the text instructions
+        if (images && images.length > 0) {
+            for (const img of images) {
+                // Add label as text before each image for context
+                if (img.label) {
+                    content.push({ type: 'text', text: `[${img.label}]` });
+                }
+                content.push({
+                    type: 'image',
+                    source: {
+                        type: 'base64',
+                        media_type: img.mediaType,
+                        data: img.base64
+                    }
+                });
+            }
+        }
+        // Add the main text prompt
+        content.push({ type: 'text', text: prompt });
+        return content;
+    }
+    /**
+     * Create a generator that yields the initial user message with optional images
+     */
+    async *createMessageGenerator(prompt, images) {
+        const content = this.buildContentBlocks(prompt, images);
+        yield {
+            type: 'user',
+            message: {
+                role: 'user',
+                content
+            },
+            parent_tool_use_id: null,
+            session_id: '' // SDK will assign the actual session ID
+        };
+    }
+    async createSession(ws, projectDir) {
+        const send = (data) => {
+            if (ws.readyState === ws.OPEN) {
+                ws.send(JSON.stringify(data));
+            }
+        };
+        // Track session ID for conversation continuity (stable V1 resume feature)
+        let sessionId;
+        return {
+            send: async (msg) => {
+                try {
+                    // Use generator function to pass multimodal content (text + images)
+                    const messageGenerator = this.createMessageGenerator(msg.prompt, msg.images);
+                    const response = query({
+                        prompt: messageGenerator,
+                        options: {
+                            model: this.model,
+                            cwd: projectDir,
+                            permissionMode: 'acceptEdits', // Auto-approve file edits (no interactive terminal)
+                            // Resume previous session if we have a session ID (enables follow-up conversations)
+                            ...(sessionId && { resume: sessionId }),
+                            systemPrompt: `You are a UI debugging assistant. When given annotations about UI elements,
+you search for the corresponding code using the selectors and HTML context provided,
+then make the requested changes. Be thorough in finding the right files and making precise edits.
+
+When screenshots are provided, use them to:
+- Better understand the visual context and styling of the elements
+- Identify the exact appearance that needs to be changed
+- Verify you're targeting the correct element based on its visual representation
+
+IMPORTANT: Format all responses using markdown:
+- Use **bold** for emphasis on important points
+- Use numbered lists for sequential steps (1. 2. 3.)
+- Use bullet points for non-sequential items
+- Use code blocks with \`\`\`language syntax for code examples
+- Use inline \`code\` for file paths, selectors, and technical terms`
+                        }
+                    });
+                    // Process streaming response
+                    for await (const message of response) {
+                        switch (message.type) {
+                            case 'system':
+                                // Capture session ID from init message for follow-up conversations
+                                if ('subtype' in message && message.subtype === 'init') {
+                                    sessionId = message.session_id;
+                                }
+                                break;
+                            case 'assistant':
+                                // Handle assistant message - extract text from BetaMessage content
+                                if (message.message?.content) {
+                                    for (const block of message.message.content) {
+                                        if (block.type === 'text') {
+                                            send({ type: 'delta', content: block.text });
+                                        }
+                                    }
+                                }
+                                break;
+                            case 'stream_event':
+                                // Handle streaming events for real-time updates
+                                if (message.event?.type === 'content_block_delta') {
+                                    const delta = message.event.delta;
+                                    if (delta && 'text' in delta) {
+                                        send({ type: 'delta', content: delta.text });
+                                    }
+                                }
+                                break;
+                            case 'tool_progress':
+                                // Tool is being executed
+                                send({ type: 'tool_start', tool: message.tool_name });
+                                break;
+                            case 'result':
+                                // Query completed
+                                send({ type: 'idle' });
+                                break;
+                        }
+                    }
+                }
+                catch (err) {
+                    send({ type: 'error', message: err.message });
+                }
+            },
+            destroy: async () => {
+                // SDK handles session cleanup automatically
+            }
+        };
+    }
+}

package/server/dist/agents/codex.d.ts

@@ -0,0 +1,12 @@
+import type { WebSocket } from 'ws';
+import { BaseAgent } from './base.js';
+import type { AgentSession } from '../types.js';
+export declare class CodexAgent extends BaseAgent {
+    name: string;
+    model: string;
+    private codex;
+    constructor();
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    createSession(ws: WebSocket, projectDir: string): Promise<AgentSession>;
+}

package/server/dist/agents/codex.js

@@ -0,0 +1,194 @@
+// server/src/agents/codex.ts
+// Agent that uses OpenAI Codex SDK
+import { Codex } from '@openai/codex-sdk';
+import { BaseAgent } from './base.js';
+import { promises as fs } from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+import { randomUUID } from 'crypto';
+export class CodexAgent extends BaseAgent {
+    name = 'codex';
+    model;
+    codex = null;
+    constructor() {
+        super();
+        this.model = process.env.CODEX_MODEL || 'gpt-5.2-codex';
+    }
+    async start() {
+        // Initialize the Codex client
+        // Uses cached credentials from ~/.codex/auth.json (login via `codex` CLI)
+        this.codex = new Codex();
+        console.log(`✓ Codex SDK initialized (model: ${this.model})`);
+    }
+    async stop() {
+        // Codex SDK doesn't require explicit cleanup
+        this.codex = null;
+    }
+    async createSession(ws, projectDir) {
+        if (!this.codex) {
+            throw new Error('Codex client not initialized');
+        }
+        const send = (data) => {
+            if (ws.readyState === ws.OPEN) {
+                ws.send(JSON.stringify(data));
+            }
+        };
+        // Start a Codex thread with the project directory
+        const thread = this.codex.startThread({
+            workingDirectory: projectDir,
+        });
+        let abortController = null;
+        // Track temp files for cleanup on session destroy (prevents race condition)
+        const sessionTempFiles = [];
+        return {
+            send: async (msg) => {
+                try {
+                    abortController = new AbortController();
+                    const input = [];
+                    // If images are provided, save them as temp files and add to structured input
+                    if (msg.images && msg.images.length > 0) {
+                        const tempDir = os.tmpdir();
+                        for (let i = 0; i < msg.images.length; i++) {
+                            const img = msg.images[i];
+                            // Use UUID to avoid filename collisions
+                            const ext = img.mediaType.split('/')[1] || 'png';
+                            const tempPath = path.join(tempDir, `zingit-screenshot-${randomUUID()}.${ext}`);
+                            // Decode base64 to buffer with error handling
+                            let buffer;
+                            try {
+                                buffer = Buffer.from(img.base64, 'base64');
+                            }
+                            catch (decodeErr) {
+                                console.warn(`ZingIt: Failed to decode base64 for image ${i + 1}:`, decodeErr);
+                                continue; // Skip this image
+                            }
+                            // Save with restrictive permissions (owner read/write only)
+                            await fs.writeFile(tempPath, buffer, { mode: 0o600 });
+                            sessionTempFiles.push(tempPath);
+                            // Add label text before image
+                            if (img.label) {
+                                input.push({ type: 'text', text: `[${img.label}]` });
+                            }
+                            // Add image as local_image input
+                            input.push({ type: 'local_image', path: tempPath });
+                        }
+                    }
+                    // Add system instructions and main prompt
+                    const systemInstructions = `You are a UI debugging assistant. When given annotations about UI elements, search for the corresponding code using the selectors and HTML context provided, then make the requested changes.
+
+When screenshots are provided, use them to:
+- Better understand the visual context and styling of the elements
+- Identify the exact appearance that needs to be changed
+- Verify you're targeting the correct element based on its visual representation
+
+IMPORTANT: Format all responses using markdown:
+- Use **bold** for emphasis on important points
+- Use numbered lists for sequential steps (1. 2. 3.)
+- Use bullet points for non-sequential items
+- Use code blocks with \`\`\`language syntax for code examples
+- Use inline \`code\` for file paths, selectors, and technical terms
+
+`;
+                    input.push({ type: 'text', text: systemInstructions + msg.prompt });
+                    // Use runStreamed with structured input for real-time progress
+                    const { events } = await thread.runStreamed(input);
+                    for await (const event of events) {
+                        // Check if aborted
+                        if (abortController?.signal.aborted) {
+                            break;
+                        }
+                        switch (event.type) {
+                            case 'item.started':
+                                // Tool/action started
+                                if (event.item?.type) {
+                                    const toolName = getToolDisplayName(event.item);
+                                    send({ type: 'tool_start', tool: toolName });
+                                }
+                                break;
+                            case 'item.completed':
+                                // Item completed - extract content based on type
+                                if (event.item) {
+                                    switch (event.item.type) {
+                                        case 'agent_message':
+                                            // Agent's text response
+                                            send({ type: 'delta', content: event.item.text + '\n' });
+                                            break;
+                                        case 'reasoning':
+                                            // Optional: show reasoning
+                                            send({ type: 'delta', content: `\n*[Reasoning]* ${event.item.text}\n` });
+                                            break;
+                                        case 'command_execution':
+                                            // Command was executed
+                                            send({ type: 'delta', content: `\n$ ${event.item.command}\n${event.item.aggregated_output}\n` });
+                                            break;
+                                        case 'file_change':
+                                            // Files were changed
+                                            const files = event.item.changes.map(c => `${c.kind}: ${c.path}`).join(', ');
+                                            send({ type: 'delta', content: `\n*[Files changed]* ${files}\n` });
+                                            break;
+                                    }
+                                    send({ type: 'tool_end', tool: event.item.type });
+                                }
+                                break;
+                            case 'turn.completed':
+                                // Turn finished
+                                send({ type: 'idle' });
+                                break;
+                            case 'turn.failed':
+                                // Turn failed with error
+                                send({ type: 'error', message: event.error?.message || 'Codex turn failed' });
+                                break;
+                            case 'error':
+                                send({ type: 'error', message: event.message || 'Unknown Codex error' });
+                                break;
+                        }
+                    }
+                }
+                catch (err) {
+                    send({ type: 'error', message: err.message });
+                }
+                // Note: Temp files cleaned up on session destroy to avoid race condition
+            },
+            destroy: async () => {
+                try {
+                    // Abort any ongoing operation
+                    if (abortController) {
+                        abortController.abort();
+                        abortController = null;
+                    }
+                    // Thread cleanup happens automatically
+                }
+                finally {
+                    // Clean up all temp files even if abort fails
+                    for (const tempPath of sessionTempFiles) {
+                        try {
+                            await fs.unlink(tempPath);
+                        }
+                        catch (cleanupErr) {
+                            // Ignore errors (file may already be deleted)
+                            console.warn(`ZingIt: Failed to clean up temp file ${tempPath}:`, cleanupErr.message);
+                        }
+                    }
+                    sessionTempFiles.length = 0; // Clear the array
+                }
+            }
+        };
+    }
+}
+// Helper to get a readable tool name
+function getToolDisplayName(item) {
+    switch (item.type) {
+        case 'command_execution':
+            return `Running: ${item.command?.split(' ')[0] || 'command'}`;
+        case 'mcp_tool_call':
+            return `Tool: ${item.tool || 'mcp'}`;
+        case 'file_change':
+            return 'Editing files';
+        case 'web_search':
+            return 'Searching web';
+        case 'reasoning':
+            return 'Thinking...';
+        default:
+            return item.type;
+    }
+}

package/server/dist/agents/copilot.d.ts

@@ -0,0 +1,12 @@
+import type { WebSocket } from 'ws';
+import { BaseAgent } from './base.js';
+import type { AgentSession } from '../types.js';
+export declare class CopilotAgent extends BaseAgent {
+    name: string;
+    model: string;
+    private client;
+    constructor();
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    createSession(ws: WebSocket, projectDir: string): Promise<AgentSession>;
+}