@cephalization/phoenix-insight 0.2.0 → 0.4.0

package/README.md

@@ -87,7 +87,10 @@ phoenix> exit
 Create or update snapshots separately from queries:
 
 ```bash
-# Create initial snapshot
+# Create initial snapshot (explicit command)
+phoenix-insight snapshot create
+
+# Create initial snapshot (shorthand, same as 'snapshot create')
 phoenix-insight snapshot
 
 # Force refresh snapshot (ignore cache)
@@ -96,6 +99,12 @@ phoenix-insight snapshot --refresh
 # Snapshot from a specific Phoenix instance
 phoenix-insight snapshot --base-url https://phoenix.example.com --api-key your-api-key
 
+# Get the path to the latest snapshot
+phoenix-insight snapshot latest
+
+# List all available snapshots
+phoenix-insight snapshot list
+
 # Clean up local snapshots
 phoenix-insight prune
 
@@ -293,8 +302,11 @@ Creates or updates a data snapshot from Phoenix without running a query.
 
 ```bash
 phoenix-insight snapshot [options]
+phoenix-insight snapshot create [options]
 ```
 
+Note: `phoenix-insight snapshot` (without subcommand) is equivalent to `phoenix-insight snapshot create` for backward compatibility.
+
 | Option             | Description                                   | Default                 | Example                                         |
 | ------------------ | --------------------------------------------- | ----------------------- | ----------------------------------------------- |
 | `--config <path>`  | Custom config file path                       | `~/.phoenix-insight/config.json` | `--config ./my-config.json`            |
@@ -304,6 +316,76 @@ phoenix-insight snapshot [options]
 | `--limit <n>`      | Maximum spans to fetch per project            | `1000`                  | `phoenix-insight snapshot --limit 5000`         |
 | `--trace`          | Enable tracing of snapshot operations         | `false`                 | `phoenix-insight snapshot --trace`              |
 
+### Snapshot Create Subcommand
+
+Explicitly creates a new snapshot from Phoenix data. This is the preferred way to create snapshots.
+
+```bash
+phoenix-insight snapshot create
+```
+
+Same options as `phoenix-insight snapshot`. Use `snapshot create` for clarity in scripts and automation.
+
+### Snapshot Latest Command
+
+Prints the absolute path to the latest snapshot directory.
+
+```bash
+phoenix-insight snapshot latest
+```
+
+Outputs the path to stdout with no decoration (just the path). Exit code 0 on success, exit code 1 if no snapshots exist.
+
+**Example usage:**
+
+```bash
+# Get the latest snapshot path
+phoenix-insight snapshot latest
+# Output: /Users/you/.phoenix-insight/snapshots/1704067200000-abc123/phoenix
+
+# Use in scripts
+SNAPSHOT_PATH=$(phoenix-insight snapshot latest)
+ls "$SNAPSHOT_PATH"
+
+# Check if snapshots exist
+if phoenix-insight snapshot latest > /dev/null 2>&1; then
+  echo "Snapshots available"
+else
+  echo "No snapshots found"
+fi
+```
+
+### Snapshot List Command
+
+Lists all available snapshots with their timestamps.
+
+```bash
+phoenix-insight snapshot list
+```
+
+Outputs one snapshot per line in the format `<timestamp> <path>` where timestamp is ISO 8601. Most recent first. Exit code 0 even if empty (just prints nothing).
+
+**Example usage:**
+
+```bash
+# List all snapshots
+phoenix-insight snapshot list
+# Output:
+# 2024-01-01T12:30:00.000Z /Users/you/.phoenix-insight/snapshots/1704113400000-abc123/phoenix
+# 2024-01-01T10:00:00.000Z /Users/you/.phoenix-insight/snapshots/1704104400000-def456/phoenix
+
+# Count snapshots
+phoenix-insight snapshot list | wc -l
+
+# Get oldest snapshot path
+phoenix-insight snapshot list | tail -1 | cut -d' ' -f2
+
+# Process snapshots in a script
+phoenix-insight snapshot list | while read timestamp path; do
+  echo "Snapshot from $timestamp at $path"
+done
+```
+
 ### Prune Command
 
 Deletes the local snapshot directory to free up disk space.

package/dist/cli.js

@@ -7,6 +7,7 @@ import * as os from "node:os";
 import { createSandboxMode, createLocalMode } from "./modes/index.js";
 import { createInsightAgent, runOneShotQuery } from "./agent/index.js";
 import { createSnapshot, createIncrementalSnapshot, createPhoenixClient, PhoenixClientError, } from "./snapshot/index.js";
+import { getLatestSnapshot, listSnapshots } from "./snapshot/utils.js";
 import { AgentProgress } from "./progress.js";
 import { initializeObservability, shutdownObservability, } from "./observability/index.js";
 import { initializeConfig, getConfig } from "./config/index.js";
@@ -182,10 +183,11 @@ Examples:
     // Initialize config singleton before any command runs
     await initializeConfig(cliArgs);
 });
-program
-    .command("snapshot")
-    .description("Create a snapshot of Phoenix data")
-    .action(async () => {
+/**
+ * Shared logic for creating a snapshot.
+ * Used by both 'phoenix-insight snapshot' and 'phoenix-insight snapshot create'.
+ */
+async function executeSnapshotCreate() {
     const config = getConfig();
     // Initialize observability if trace is enabled in config
     if (config.trace) {
@@ -216,6 +218,61 @@ program
     catch (error) {
         handleError(error, "creating snapshot");
     }
+}
+// Create snapshot command group
+const snapshotCmd = program
+    .command("snapshot")
+    .description("Snapshot management commands");
+// Default action for 'phoenix-insight snapshot' (backward compatibility alias for 'snapshot create')
+snapshotCmd.action(async () => {
+    await executeSnapshotCreate();
+});
+// Subcommand: snapshot create (explicit create command)
+snapshotCmd
+    .command("create")
+    .description("Create a new snapshot from Phoenix data")
+    .action(async () => {
+    await executeSnapshotCreate();
+});
+// Subcommand: snapshot latest
+snapshotCmd
+    .command("latest")
+    .description("Print the absolute path to the latest snapshot directory")
+    .action(async () => {
+    try {
+        const latestSnapshot = await getLatestSnapshot();
+        if (!latestSnapshot) {
+            console.error("No snapshots found");
+            process.exit(1);
+        }
+        // Print only the path to stdout, no decoration
+        console.log(latestSnapshot.path);
+    }
+    catch (error) {
+        console.error(`Error: ${error instanceof Error ? error.message : String(error)}`);
+        process.exit(1);
+    }
+});
+// Subcommand: snapshot list
+snapshotCmd
+    .command("list")
+    .description("List all available snapshots with their timestamps")
+    .action(async () => {
+    try {
+        const snapshots = await listSnapshots();
+        // Print each snapshot: <timestamp> <path>
+        // Most recent first (already sorted by listSnapshots)
+        for (const snapshot of snapshots) {
+            // Format timestamp as ISO 8601
+            const isoTimestamp = snapshot.timestamp.toISOString();
+            console.log(`${isoTimestamp} ${snapshot.path}`);
+        }
+        // Exit code 0 even if empty (just print nothing)
+    }
+    catch (error) {
+        console.error(`Error: ${error instanceof Error ? error.message : String(error)}`);
+        process.exit(1);
+    }
 });
 program
     .command("help")

package/dist/snapshot/context.js

@@ -1,16 +1,160 @@
+// =============================================================================
+// Static Section Templates
+// =============================================================================
+/**
+ * Quick Start section for external agents - appears at the top for discoverability
+ */
+const QUICK_START_SECTION = `## Quick Start for External Agents
+
+This is a **read-only snapshot** of Phoenix observability data. You cannot modify this data.
+
+### Key Files to Start With
+
+| File | Description |
+|------|-------------|
+| \`/phoenix/projects/index.jsonl\` | List of all projects with traces |
+| \`/phoenix/datasets/index.jsonl\` | List of all datasets |
+| \`/phoenix/experiments/index.jsonl\` | List of all experiments |
+| \`/phoenix/prompts/index.jsonl\` | List of all prompts |
+
+### How to Parse Each File Format
+
+**JSONL files** (\`.jsonl\`): One JSON object per line
+\`\`\`bash
+# Read all lines as a JSON array
+cat /phoenix/projects/index.jsonl | jq -s '.'
+
+# Process each line individually
+while read -r line; do echo "$line" | jq '.name'; done < /phoenix/projects/index.jsonl
+
+# Get first N items
+head -n 5 /phoenix/projects/index.jsonl | jq -s '.'
+\`\`\`
+
+**JSON files** (\`.json\`): Standard JSON format
+\`\`\`bash
+# Read and pretty-print
+cat /phoenix/projects/my-project/metadata.json | jq '.'
+
+# Extract specific field
+cat /phoenix/projects/my-project/metadata.json | jq '.name'
+\`\`\`
+
+**Markdown files** (\`.md\`): Plain text prompt templates
+\`\`\`bash
+# Read prompt template
+cat /phoenix/prompts/my-prompt/versions/v1.md
+\`\`\`
+
+### Common Operations
+
+\`\`\`bash
+# List all project names
+cat /phoenix/projects/index.jsonl | jq -r '.name'
+
+# Count spans in a project
+wc -l < /phoenix/projects/my-project/spans/index.jsonl
+
+# Find spans with errors
+cat /phoenix/projects/my-project/spans/index.jsonl | jq 'select(.status_code == "ERROR")'
+
+# Get dataset examples
+cat /phoenix/datasets/my-dataset/examples.jsonl | jq -s '.' | head -n 100
+
+# Search across all files
+grep -r "error" /phoenix/
+\`\`\``;
+/**
+ * Directory Structure section showing the snapshot layout
+ */
+const DIRECTORY_STRUCTURE_SECTION = `## Directory Structure
+
+\`\`\`
+/phoenix/
+  _context.md                    # This file - start here!
+  /projects/
+    index.jsonl                  # List of all projects
+    /{project_name}/
+      metadata.json              # Project details
+      /spans/
+        index.jsonl              # Span data (may be sampled)
+        metadata.json            # Span snapshot metadata
+  /datasets/
+    index.jsonl                  # List of all datasets
+    /{dataset_name}/
+      metadata.json              # Dataset details
+      examples.jsonl             # Dataset examples
+  /experiments/
+    index.jsonl                  # List of all experiments
+    /{experiment_id}/
+      metadata.json              # Experiment details
+      runs.jsonl                 # Experiment runs
+  /prompts/
+    index.jsonl                  # List of all prompts
+    /{prompt_name}/
+      metadata.json              # Prompt details
+      /versions/
+        index.jsonl              # Version list
+        /{version_id}.md         # Version template
+  /_meta/
+    snapshot.json                # Snapshot metadata
+\`\`\``;
+/**
+ * What You Can Do section describing available operations
+ */
+const WHAT_YOU_CAN_DO_SECTION = `## What You Can Do
+
+- **Explore**: ls, cat, grep, find, jq, awk, sed
+- **Fetch more data**: \`px-fetch-more spans --project <name> --limit 500\`
+- **Fetch specific trace**: \`px-fetch-more trace --trace-id <id>\``;
+/**
+ * Data Freshness section with refresh instructions
+ */
+const DATA_FRESHNESS_SECTION = `## Data Freshness
+
+This is a **read-only snapshot**. Data may have changed since capture.
+Run with \`--refresh\` to get latest data.`;
+// =============================================================================
+// Main Context Generation
+// =============================================================================
 /**
  * Generates a _context.md summary file for the Phoenix snapshot
  * This provides human and agent-readable context about what data is available
  */
 export async function generateContext(mode, metadata) {
-    const lines = [];
-    // Header
-    lines.push("# Phoenix Snapshot Context");
-    lines.push("");
     // Collect stats from the snapshot
     const stats = await collectSnapshotStats(mode);
-    // What's Here section
+    // Build the dynamic "What's Here" section
+    const whatsHereSection = buildWhatsHereSection(stats, metadata);
+    // Build the dynamic "Recent Activity" section (may be empty)
+    const recentActivitySection = buildRecentActivitySection(stats);
+    // Compose the full context document
+    const content = [
+        "# Phoenix Snapshot Context",
+        "",
+        QUICK_START_SECTION,
+        "",
+        whatsHereSection,
+        recentActivitySection,
+        DIRECTORY_STRUCTURE_SECTION,
+        "",
+        WHAT_YOU_CAN_DO_SECTION,
+        "",
+        DATA_FRESHNESS_SECTION,
+    ].join("\n");
+    // Write the context file
+    await mode.writeFile("/phoenix/_context.md", content);
+}
+// =============================================================================
+// Dynamic Section Builders
+// =============================================================================
+/**
+ * Builds the "What's Here" section with project/dataset/experiment/prompt summaries
+ */
+function buildWhatsHereSection(stats, metadata) {
+    const lines = [];
     lines.push("## What's Here");
+    lines.push("");
     // Projects summary
     if (stats.projects.length > 0) {
         const projectSummary = stats.projects
@@ -57,67 +201,29 @@ export async function generateContext(mode, metadata) {
     // Snapshot metadata
     lines.push(`- **Snapshot**: Created ${formatRelativeTime(metadata.snapshotTime)} from ${metadata.phoenixUrl}`);
     lines.push("");
-    // Recent Activity section (if we have recent data)
-    const recentActivity = getRecentActivity(stats);
-    if (recentActivity.length > 0) {
-        lines.push("## Recent Activity");
-        for (const activity of recentActivity) {
-            lines.push(`- ${activity}`);
-        }
-        lines.push("");
+    return lines.join("\n");
+}
+/**
+ * Builds the "Recent Activity" section if there are recent updates
+ * Returns an empty string if no recent activity
+ */
+function buildRecentActivitySection(stats) {
+    const activities = getRecentActivity(stats);
+    if (activities.length === 0) {
+        return "";
     }
-    // What You Can Do section
-    lines.push("## What You Can Do");
-    lines.push("- **Explore**: ls, cat, grep, find, jq, awk, sed");
-    lines.push("- **Fetch more data**: `px-fetch-more spans --project <name> --limit 500`");
-    lines.push("- **Fetch specific trace**: `px-fetch-more trace --trace-id <id>`");
-    lines.push("");
-    // Data Freshness section
-    lines.push("## Data Freshness");
-    lines.push("This is a **read-only snapshot**. Data may have changed since capture.");
-    lines.push("Run with `--refresh` to get latest data.");
+    const lines = [];
+    lines.push("## Recent Activity");
     lines.push("");
-    // File Formats section
-    lines.push("## File Formats");
-    lines.push("- `.jsonl` files: One JSON object per line, use `jq -s` to parse as array");
-    lines.push("- `.json` files: Standard JSON");
-    lines.push("- `.md` files: Markdown (prompt templates)");
+    for (const activity of activities) {
+        lines.push(`- ${activity}`);
+    }
     lines.push("");
-    // Directory Structure section
-    lines.push("## Directory Structure");
-    lines.push("```");
-    lines.push("/phoenix/");
-    lines.push("  _context.md                    # This file");
-    lines.push("  /projects/");
-    lines.push("    index.jsonl                  # List of all projects");
-    lines.push("    /{project_name}/");
-    lines.push("      metadata.json              # Project details");
-    lines.push("      /spans/");
-    lines.push("        index.jsonl              # Span data (may be sampled)");
-    lines.push("        metadata.json            # Span snapshot metadata");
-    lines.push("  /datasets/");
-    lines.push("    index.jsonl                  # List of all datasets");
-    lines.push("    /{dataset_name}/");
-    lines.push("      metadata.json              # Dataset details");
-    lines.push("      examples.jsonl             # Dataset examples");
-    lines.push("  /experiments/");
-    lines.push("    index.jsonl                  # List of all experiments");
-    lines.push("    /{experiment_id}/");
-    lines.push("      metadata.json              # Experiment details");
-    lines.push("      runs.jsonl                 # Experiment runs");
-    lines.push("  /prompts/");
-    lines.push("    index.jsonl                  # List of all prompts");
-    lines.push("    /{prompt_name}/");
-    lines.push("      metadata.json              # Prompt details");
-    lines.push("      /versions/");
-    lines.push("        index.jsonl              # Version list");
-    lines.push("        /{version_id}.md         # Version template");
-    lines.push("  /_meta/");
-    lines.push("    snapshot.json                # Snapshot metadata");
-    lines.push("```");
-    // Write the context file
-    await mode.writeFile("/phoenix/_context.md", lines.join("\n"));
+    return lines.join("\n");
 }
+// =============================================================================
+// Data Collection
+// =============================================================================
 /**
  * Collects statistics from the snapshot filesystem
  */
@@ -259,6 +365,9 @@ async function collectSnapshotStats(mode) {
     }
     return result;
 }
+// =============================================================================
+// Helper Functions
+// =============================================================================
 /**
  * Determines the status of an experiment based on its run counts
  */

package/dist/snapshot/spans.js

@@ -1,4 +1,17 @@
 import { withErrorHandling } from "./client.js";
+/**
+ * Debug logger that respects the debug flag or DEBUG environment variable
+ */
+function createDebugLogger(debug) {
+    const isDebugEnabled = debug ?? !!process.env.DEBUG;
+    return {
+        log: (message) => {
+            if (isDebugEnabled) {
+                console.log(`[snapshotSpans] ${message}`);
+            }
+        },
+    };
+}
 /**
  * Fetches spans for all projects and writes them to the snapshot
  *
@@ -7,11 +20,15 @@ import { withErrorHandling } from "./client.js";
  * @param options - Options for filtering and limiting spans
  */
 export async function snapshotSpans(client, mode, options = {}) {
-    const { startTime, endTime, spansPerProject = 1000 } = options;
+    const { startTime, endTime, spansPerProject = 1000, debug } = options;
+    const logger = createDebugLogger(debug);
     // Read projects index to get project names
-    const projectsIndexContent = await mode.exec("cat /phoenix/projects/index.jsonl");
+    // Use relative path so it works with the cwd set by the execution mode
+    logger.log("Reading projects index from projects/index.jsonl");
+    const projectsIndexContent = await mode.exec("cat projects/index.jsonl");
     if (!projectsIndexContent.stdout) {
         // No projects, nothing to do
+        logger.log("No projects found in index, skipping span fetch");
         return;
     }
     const projectNames = projectsIndexContent.stdout
@@ -22,9 +39,11 @@ export async function snapshotSpans(client, mode, options = {}) {
         const project = JSON.parse(line);
         return project.name;
     });
+    logger.log(`Found ${projectNames.length} project(s): ${projectNames.join(", ")}`);
     // Fetch spans for each project
     for (const projectName of projectNames) {
         await withErrorHandling(async () => {
+            logger.log(`Starting span fetch for project: ${projectName}`);
             const spans = [];
             let cursor = null;
             let totalFetched = 0;
@@ -63,10 +82,15 @@ export async function snapshotSpans(client, mode, options = {}) {
                     break;
                 }
             }
+            logger.log(`Completed span fetch for project ${projectName}: ${spans.length} span(s) fetched`);
             // Write spans to JSONL file
+            const spansFilePath = `/phoenix/projects/${projectName}/spans/index.jsonl`;
+            logger.log(`Writing spans to ${spansFilePath}`);
             const jsonlContent = spans.map((span) => JSON.stringify(span)).join("\n");
-            await mode.writeFile(`/phoenix/projects/${projectName}/spans/index.jsonl`, jsonlContent);
+            await mode.writeFile(spansFilePath, jsonlContent);
             // Write metadata about the spans snapshot
+            const metadataFilePath = `/phoenix/projects/${projectName}/spans/metadata.json`;
+            logger.log(`Writing metadata to ${metadataFilePath}`);
             const metadata = {
                 project: projectName,
                 spanCount: spans.length,
@@ -74,7 +98,7 @@ export async function snapshotSpans(client, mode, options = {}) {
                 endTime: endTime || null,
                 snapshotTime: new Date().toISOString(),
             };
-            await mode.writeFile(`/phoenix/projects/${projectName}/spans/metadata.json`, JSON.stringify(metadata, null, 2));
+            await mode.writeFile(metadataFilePath, JSON.stringify(metadata, null, 2));
         }, `fetching spans for project ${projectName}`);
     }
 }

package/dist/snapshot/utils.js

@@ -0,0 +1,112 @@
+/**
+ * Snapshot discovery utilities
+ *
+ * Functions for listing and finding snapshots in the local filesystem.
+ */
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+import * as os from "node:os";
+/**
+ * Get the base snapshots directory path
+ */
+export function getSnapshotsDir() {
+    return path.join(os.homedir(), ".phoenix-insight", "snapshots");
+}
+/**
+ * Parse a snapshot directory name to extract timestamp
+ *
+ * Directory names are in format: `<timestamp>-<random>` where timestamp is Date.now()
+ * Example: "1704067200000-abc123" -> Date(2024-01-01T00:00:00.000Z)
+ *
+ * @param dirName - The directory name to parse
+ * @returns The parsed timestamp as Date, or null if invalid
+ */
+function parseSnapshotDirName(dirName) {
+    // Format: <timestamp>-<random>
+    const match = dirName.match(/^(\d+)-[\w]+$/);
+    if (!match || !match[1]) {
+        return null;
+    }
+    const timestamp = parseInt(match[1], 10);
+    if (isNaN(timestamp) || timestamp <= 0) {
+        return null;
+    }
+    const date = new Date(timestamp);
+    // Validate the date is reasonable (between year 2000 and year 3000)
+    // Use UTC year to avoid timezone issues
+    const year = date.getUTCFullYear();
+    if (year < 2000 || year > 3000) {
+        return null;
+    }
+    return date;
+}
+/**
+ * List all available snapshots
+ *
+ * Scans the snapshots directory and returns information about each valid snapshot.
+ * Results are sorted by timestamp descending (most recent first).
+ *
+ * @returns Array of snapshot info objects, sorted by timestamp descending
+ */
+export async function listSnapshots() {
+    const snapshotsDir = getSnapshotsDir();
+    // Check if snapshots directory exists
+    try {
+        await fs.access(snapshotsDir);
+    }
+    catch {
+        // Directory doesn't exist - return empty array
+        return [];
+    }
+    // Read directory contents
+    let entries;
+    try {
+        entries = await fs.readdir(snapshotsDir);
+    }
+    catch {
+        // Cannot read directory - return empty array
+        return [];
+    }
+    // Filter and parse valid snapshot directories
+    const snapshots = [];
+    for (const entry of entries) {
+        const timestamp = parseSnapshotDirName(entry);
+        if (!timestamp) {
+            // Invalid directory name format - skip
+            continue;
+        }
+        const snapshotPath = path.join(snapshotsDir, entry, "phoenix");
+        // Verify the phoenix subdirectory exists
+        try {
+            const stat = await fs.stat(snapshotPath);
+            if (!stat.isDirectory()) {
+                continue;
+            }
+        }
+        catch {
+            // Phoenix subdirectory doesn't exist or can't be accessed - skip
+            continue;
+        }
+        snapshots.push({
+            path: snapshotPath,
+            timestamp,
+            id: entry,
+        });
+    }
+    // Sort by timestamp descending (most recent first)
+    snapshots.sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime());
+    return snapshots;
+}
+/**
+ * Get the latest (most recent) snapshot
+ *
+ * @returns The most recent snapshot info, or null if no snapshots exist
+ */
+export async function getLatestSnapshot() {
+    const snapshots = await listSnapshots();
+    if (snapshots.length === 0) {
+        return null;
+    }
+    // First element is the most recent due to descending sort
+    return snapshots[0] ?? null;
+}

package/dist/tsconfig.esm.tsbuildinfo

@@ -1 +1 @@
-{"root":["../src/cli.ts","../src/index.ts","../src/progress.ts","../src/agent/index.ts","../src/commands/index.ts","../src/commands/px-fetch-more-spans.ts","../src/commands/px-fetch-more-trace.ts","../src/config/index.ts","../src/config/loader.ts","../src/config/schema.ts","../src/modes/index.ts","../src/modes/local.ts","../src/modes/sandbox.ts","../src/modes/types.ts","../src/observability/index.ts","../src/prompts/index.ts","../src/prompts/system.ts","../src/snapshot/client.ts","../src/snapshot/context.ts","../src/snapshot/datasets.ts","../src/snapshot/experiments.ts","../src/snapshot/index.ts","../src/snapshot/projects.ts","../src/snapshot/prompts.ts","../src/snapshot/spans.ts"],"version":"5.9.3"}
+{"root":["../src/cli.ts","../src/index.ts","../src/progress.ts","../src/agent/index.ts","../src/commands/index.ts","../src/commands/px-fetch-more-spans.ts","../src/commands/px-fetch-more-trace.ts","../src/config/index.ts","../src/config/loader.ts","../src/config/schema.ts","../src/modes/index.ts","../src/modes/local.ts","../src/modes/sandbox.ts","../src/modes/types.ts","../src/observability/index.ts","../src/prompts/index.ts","../src/prompts/system.ts","../src/snapshot/client.ts","../src/snapshot/context.ts","../src/snapshot/datasets.ts","../src/snapshot/experiments.ts","../src/snapshot/index.ts","../src/snapshot/projects.ts","../src/snapshot/prompts.ts","../src/snapshot/spans.ts","../src/snapshot/utils.ts"],"version":"5.9.3"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cephalization/phoenix-insight",
-  "version": "0.2.0",
+  "version": "0.4.0",
   "description": "A CLI for Arize AI Phoenix data analysis with AI agents",
   "type": "module",
   "exports": {
@@ -53,7 +53,9 @@
   },
   "devDependencies": {
     "@changesets/cli": "^2.29.8",
+    "@faker-js/faker": "^10.2.0",
     "@types/node": "^18.19.0",
+    "msw": "^2.12.7",
     "rimraf": "^5.0.10",
     "tsc-alias": "^1.8.11",
     "tsx": "^4.21.0",

package/src/cli.ts

@@ -13,6 +13,7 @@ import {
   createPhoenixClient,
   PhoenixClientError,
 } from "./snapshot/index.js";
+import { getLatestSnapshot, listSnapshots } from "./snapshot/utils.js";
 import type { ExecutionMode } from "./modes/types.js";
 import type { PhoenixInsightAgentConfig } from "./agent/index.js";
 import { AgentProgress } from "./progress.js";
@@ -218,44 +219,111 @@ Examples:
     await initializeConfig(cliArgs);
   });
 
-program
+/**
+ * Shared logic for creating a snapshot.
+ * Used by both 'phoenix-insight snapshot' and 'phoenix-insight snapshot create'.
+ */
+async function executeSnapshotCreate(): Promise<void> {
+  const config = getConfig();
+
+  // Initialize observability if trace is enabled in config
+  if (config.trace) {
+    initializeObservability({
+      enabled: true,
+      baseUrl: config.baseUrl,
+      apiKey: config.apiKey,
+      projectName: "phoenix-insight-snapshot",
+      debug: !!process.env.DEBUG,
+    });
+  }
+
+  try {
+    // Determine the execution mode
+    const mode: ExecutionMode = await createLocalMode();
+
+    // Create snapshot with config values
+    const snapshotOptions = {
+      baseURL: config.baseUrl,
+      apiKey: config.apiKey,
+      spansPerProject: config.limit,
+      showProgress: true,
+    };
+
+    await createSnapshot(mode, snapshotOptions);
+
+    // Cleanup
+    await mode.cleanup();
+
+    // Shutdown observability if enabled
+    await shutdownObservability();
+  } catch (error) {
+    handleError(error, "creating snapshot");
+  }
+}
+
+// Create snapshot command group
+const snapshotCmd = program
   .command("snapshot")
-  .description("Create a snapshot of Phoenix data")
-  .action(async () => {
-    const config = getConfig();
+  .description("Snapshot management commands");
 
-    // Initialize observability if trace is enabled in config
-    if (config.trace) {
-      initializeObservability({
-        enabled: true,
-        baseUrl: config.baseUrl,
-        apiKey: config.apiKey,
-        projectName: "phoenix-insight-snapshot",
-        debug: !!process.env.DEBUG,
-      });
-    }
+// Default action for 'phoenix-insight snapshot' (backward compatibility alias for 'snapshot create')
+snapshotCmd.action(async () => {
+  await executeSnapshotCreate();
+});
 
+// Subcommand: snapshot create (explicit create command)
+snapshotCmd
+  .command("create")
+  .description("Create a new snapshot from Phoenix data")
+  .action(async () => {
+    await executeSnapshotCreate();
+  });
+
+// Subcommand: snapshot latest
+snapshotCmd
+  .command("latest")
+  .description("Print the absolute path to the latest snapshot directory")
+  .action(async () => {
     try {
-      // Determine the execution mode
-      const mode: ExecutionMode = await createLocalMode();
+      const latestSnapshot = await getLatestSnapshot();
 
-      // Create snapshot with config values
-      const snapshotOptions = {
-        baseURL: config.baseUrl,
-        apiKey: config.apiKey,
-        spansPerProject: config.limit,
-        showProgress: true,
-      };
+      if (!latestSnapshot) {
+        console.error("No snapshots found");
+        process.exit(1);
+      }
 
-      await createSnapshot(mode, snapshotOptions);
+      // Print only the path to stdout, no decoration
+      console.log(latestSnapshot.path);
+    } catch (error) {
+      console.error(
+        `Error: ${error instanceof Error ? error.message : String(error)}`
+      );
+      process.exit(1);
+    }
+  });
 
-      // Cleanup
-      await mode.cleanup();
+// Subcommand: snapshot list
+snapshotCmd
+  .command("list")
+  .description("List all available snapshots with their timestamps")
+  .action(async () => {
+    try {
+      const snapshots = await listSnapshots();
+
+      // Print each snapshot: <timestamp> <path>
+      // Most recent first (already sorted by listSnapshots)
+      for (const snapshot of snapshots) {
+        // Format timestamp as ISO 8601
+        const isoTimestamp = snapshot.timestamp.toISOString();
+        console.log(`${isoTimestamp} ${snapshot.path}`);
+      }
 
-      // Shutdown observability if enabled
-      await shutdownObservability();
+      // Exit code 0 even if empty (just print nothing)
     } catch (error) {
-      handleError(error, "creating snapshot");
+      console.error(
+        `Error: ${error instanceof Error ? error.message : String(error)}`
+      );
+      process.exit(1);
     }
   });
 

package/src/snapshot/context.ts

@@ -39,6 +39,131 @@ interface PromptInfo {
   updatedAt?: string;
 }
 
+// =============================================================================
+// Static Section Templates
+// =============================================================================
+
+/**
+ * Quick Start section for external agents - appears at the top for discoverability
+ */
+const QUICK_START_SECTION = `## Quick Start for External Agents
+
+This is a **read-only snapshot** of Phoenix observability data. You cannot modify this data.
+
+### Key Files to Start With
+
+| File | Description |
+|------|-------------|
+| \`/phoenix/projects/index.jsonl\` | List of all projects with traces |
+| \`/phoenix/datasets/index.jsonl\` | List of all datasets |
+| \`/phoenix/experiments/index.jsonl\` | List of all experiments |
+| \`/phoenix/prompts/index.jsonl\` | List of all prompts |
+
+### How to Parse Each File Format
+
+**JSONL files** (\`.jsonl\`): One JSON object per line
+\`\`\`bash
+# Read all lines as a JSON array
+cat /phoenix/projects/index.jsonl | jq -s '.'
+
+# Process each line individually
+while read -r line; do echo "$line" | jq '.name'; done < /phoenix/projects/index.jsonl
+
+# Get first N items
+head -n 5 /phoenix/projects/index.jsonl | jq -s '.'
+\`\`\`
+
+**JSON files** (\`.json\`): Standard JSON format
+\`\`\`bash
+# Read and pretty-print
+cat /phoenix/projects/my-project/metadata.json | jq '.'
+
+# Extract specific field
+cat /phoenix/projects/my-project/metadata.json | jq '.name'
+\`\`\`
+
+**Markdown files** (\`.md\`): Plain text prompt templates
+\`\`\`bash
+# Read prompt template
+cat /phoenix/prompts/my-prompt/versions/v1.md
+\`\`\`
+
+### Common Operations
+
+\`\`\`bash
+# List all project names
+cat /phoenix/projects/index.jsonl | jq -r '.name'
+
+# Count spans in a project
+wc -l < /phoenix/projects/my-project/spans/index.jsonl
+
+# Find spans with errors
+cat /phoenix/projects/my-project/spans/index.jsonl | jq 'select(.status_code == "ERROR")'
+
+# Get dataset examples
+cat /phoenix/datasets/my-dataset/examples.jsonl | jq -s '.' | head -n 100
+
+# Search across all files
+grep -r "error" /phoenix/
+\`\`\``;
+
+/**
+ * Directory Structure section showing the snapshot layout
+ */
+const DIRECTORY_STRUCTURE_SECTION = `## Directory Structure
+
+\`\`\`
+/phoenix/
+  _context.md                    # This file - start here!
+  /projects/
+    index.jsonl                  # List of all projects
+    /{project_name}/
+      metadata.json              # Project details
+      /spans/
+        index.jsonl              # Span data (may be sampled)
+        metadata.json            # Span snapshot metadata
+  /datasets/
+    index.jsonl                  # List of all datasets
+    /{dataset_name}/
+      metadata.json              # Dataset details
+      examples.jsonl             # Dataset examples
+  /experiments/
+    index.jsonl                  # List of all experiments
+    /{experiment_id}/
+      metadata.json              # Experiment details
+      runs.jsonl                 # Experiment runs
+  /prompts/
+    index.jsonl                  # List of all prompts
+    /{prompt_name}/
+      metadata.json              # Prompt details
+      /versions/
+        index.jsonl              # Version list
+        /{version_id}.md         # Version template
+  /_meta/
+    snapshot.json                # Snapshot metadata
+\`\`\``;
+
+/**
+ * What You Can Do section describing available operations
+ */
+const WHAT_YOU_CAN_DO_SECTION = `## What You Can Do
+
+- **Explore**: ls, cat, grep, find, jq, awk, sed
+- **Fetch more data**: \`px-fetch-more spans --project <name> --limit 500\`
+- **Fetch specific trace**: \`px-fetch-more trace --trace-id <id>\``;
+
+/**
+ * Data Freshness section with refresh instructions
+ */
+const DATA_FRESHNESS_SECTION = `## Data Freshness
+
+This is a **read-only snapshot**. Data may have changed since capture.
+Run with \`--refresh\` to get latest data.`;
+
+// =============================================================================
+// Main Context Generation
+// =============================================================================
+
 /**
  * Generates a _context.md summary file for the Phoenix snapshot
  * This provides human and agent-readable context about what data is available
@@ -47,17 +172,54 @@ export async function generateContext(
   mode: ExecutionMode,
   metadata: ContextMetadata
 ): Promise<void> {
-  const lines: string[] = [];
-
-  // Header
-  lines.push("# Phoenix Snapshot Context");
-  lines.push("");
-
   // Collect stats from the snapshot
   const stats = await collectSnapshotStats(mode);
 
-  // What's Here section
+  // Build the dynamic "What's Here" section
+  const whatsHereSection = buildWhatsHereSection(stats, metadata);
+
+  // Build the dynamic "Recent Activity" section (may be empty)
+  const recentActivitySection = buildRecentActivitySection(stats);
+
+  // Compose the full context document
+  const content = [
+    "# Phoenix Snapshot Context",
+    "",
+    QUICK_START_SECTION,
+    "",
+    whatsHereSection,
+    recentActivitySection,
+    DIRECTORY_STRUCTURE_SECTION,
+    "",
+    WHAT_YOU_CAN_DO_SECTION,
+    "",
+    DATA_FRESHNESS_SECTION,
+  ].join("\n");
+
+  // Write the context file
+  await mode.writeFile("/phoenix/_context.md", content);
+}
+
+// =============================================================================
+// Dynamic Section Builders
+// =============================================================================
+
+/**
+ * Builds the "What's Here" section with project/dataset/experiment/prompt summaries
+ */
+function buildWhatsHereSection(
+  stats: {
+    projects: ProjectStats[];
+    datasets: DatasetInfo[];
+    experiments: ExperimentInfo[];
+    prompts: PromptInfo[];
+  },
+  metadata: ContextMetadata
+): string {
+  const lines: string[] = [];
+
   lines.push("## What's Here");
+  lines.push("");
 
   // Projects summary
   if (stats.projects.length > 0) {
@@ -115,81 +277,40 @@ export async function generateContext(
   );
   lines.push("");
 
-  // Recent Activity section (if we have recent data)
-  const recentActivity = getRecentActivity(stats);
-  if (recentActivity.length > 0) {
-    lines.push("## Recent Activity");
-    for (const activity of recentActivity) {
-      lines.push(`- ${activity}`);
-    }
-    lines.push("");
-  }
+  return lines.join("\n");
+}
 
-  // What You Can Do section
-  lines.push("## What You Can Do");
-  lines.push("- **Explore**: ls, cat, grep, find, jq, awk, sed");
-  lines.push(
-    "- **Fetch more data**: `px-fetch-more spans --project <name> --limit 500`"
-  );
-  lines.push(
-    "- **Fetch specific trace**: `px-fetch-more trace --trace-id <id>`"
-  );
-  lines.push("");
+/**
+ * Builds the "Recent Activity" section if there are recent updates
+ * Returns an empty string if no recent activity
+ */
+function buildRecentActivitySection(stats: {
+  projects: ProjectStats[];
+  datasets: DatasetInfo[];
+  experiments: ExperimentInfo[];
+  prompts: PromptInfo[];
+}): string {
+  const activities = getRecentActivity(stats);
 
-  // Data Freshness section
-  lines.push("## Data Freshness");
-  lines.push(
-    "This is a **read-only snapshot**. Data may have changed since capture."
-  );
-  lines.push("Run with `--refresh` to get latest data.");
-  lines.push("");
+  if (activities.length === 0) {
+    return "";
+  }
 
-  // File Formats section
-  lines.push("## File Formats");
-  lines.push(
-    "- `.jsonl` files: One JSON object per line, use `jq -s` to parse as array"
-  );
-  lines.push("- `.json` files: Standard JSON");
-  lines.push("- `.md` files: Markdown (prompt templates)");
+  const lines: string[] = [];
+  lines.push("## Recent Activity");
+  lines.push("");
+  for (const activity of activities) {
+    lines.push(`- ${activity}`);
+  }
   lines.push("");
 
-  // Directory Structure section
-  lines.push("## Directory Structure");
-  lines.push("```");
-  lines.push("/phoenix/");
-  lines.push("  _context.md                    # This file");
-  lines.push("  /projects/");
-  lines.push("    index.jsonl                  # List of all projects");
-  lines.push("    /{project_name}/");
-  lines.push("      metadata.json              # Project details");
-  lines.push("      /spans/");
-  lines.push("        index.jsonl              # Span data (may be sampled)");
-  lines.push("        metadata.json            # Span snapshot metadata");
-  lines.push("  /datasets/");
-  lines.push("    index.jsonl                  # List of all datasets");
-  lines.push("    /{dataset_name}/");
-  lines.push("      metadata.json              # Dataset details");
-  lines.push("      examples.jsonl             # Dataset examples");
-  lines.push("  /experiments/");
-  lines.push("    index.jsonl                  # List of all experiments");
-  lines.push("    /{experiment_id}/");
-  lines.push("      metadata.json              # Experiment details");
-  lines.push("      runs.jsonl                 # Experiment runs");
-  lines.push("  /prompts/");
-  lines.push("    index.jsonl                  # List of all prompts");
-  lines.push("    /{prompt_name}/");
-  lines.push("      metadata.json              # Prompt details");
-  lines.push("      /versions/");
-  lines.push("        index.jsonl              # Version list");
-  lines.push("        /{version_id}.md         # Version template");
-  lines.push("  /_meta/");
-  lines.push("    snapshot.json                # Snapshot metadata");
-  lines.push("```");
-
-  // Write the context file
-  await mode.writeFile("/phoenix/_context.md", lines.join("\n"));
+  return lines.join("\n");
 }
 
+// =============================================================================
+// Data Collection
+// =============================================================================
+
 /**
  * Collects statistics from the snapshot filesystem
  */
@@ -358,6 +479,10 @@ async function collectSnapshotStats(mode: ExecutionMode): Promise<{
   return result;
 }
 
+// =============================================================================
+// Helper Functions
+// =============================================================================
+
 /**
  * Determines the status of an experiment based on its run counts
  */

package/src/snapshot/spans.ts

@@ -9,6 +9,22 @@ export interface SnapshotSpansOptions {
   endTime?: Date | string | null;
   /** Maximum number of spans to fetch per project (default: 1000) */
   spansPerProject?: number;
+  /** Enable debug logging (default: uses DEBUG env var) */
+  debug?: boolean;
+}
+
+/**
+ * Debug logger that respects the debug flag or DEBUG environment variable
+ */
+function createDebugLogger(debug?: boolean) {
+  const isDebugEnabled = debug ?? !!process.env.DEBUG;
+  return {
+    log: (message: string) => {
+      if (isDebugEnabled) {
+        console.log(`[snapshotSpans] ${message}`);
+      }
+    },
+  };
 }
 
 interface SpanData {
@@ -44,14 +60,16 @@ export async function snapshotSpans(
   mode: ExecutionMode,
   options: SnapshotSpansOptions = {}
 ): Promise<void> {
-  const { startTime, endTime, spansPerProject = 1000 } = options;
+  const { startTime, endTime, spansPerProject = 1000, debug } = options;
+  const logger = createDebugLogger(debug);
 
   // Read projects index to get project names
-  const projectsIndexContent = await mode.exec(
-    "cat /phoenix/projects/index.jsonl"
-  );
+  // Use relative path so it works with the cwd set by the execution mode
+  logger.log("Reading projects index from projects/index.jsonl");
+  const projectsIndexContent = await mode.exec("cat projects/index.jsonl");
   if (!projectsIndexContent.stdout) {
     // No projects, nothing to do
+    logger.log("No projects found in index, skipping span fetch");
     return;
   }
 
@@ -64,9 +82,12 @@ export async function snapshotSpans(
       return project.name;
     });
 
+  logger.log(`Found ${projectNames.length} project(s): ${projectNames.join(", ")}`);
+
   // Fetch spans for each project
   for (const projectName of projectNames) {
     await withErrorHandling(async () => {
+      logger.log(`Starting span fetch for project: ${projectName}`);
       const spans: SpanData[] = [];
       let cursor: string | null = null;
       let totalFetched = 0;
@@ -117,14 +138,17 @@ export async function snapshotSpans(
         }
       }
 
+      logger.log(`Completed span fetch for project ${projectName}: ${spans.length} span(s) fetched`);
+
       // Write spans to JSONL file
+      const spansFilePath = `/phoenix/projects/${projectName}/spans/index.jsonl`;
+      logger.log(`Writing spans to ${spansFilePath}`);
       const jsonlContent = spans.map((span) => JSON.stringify(span)).join("\n");
-      await mode.writeFile(
-        `/phoenix/projects/${projectName}/spans/index.jsonl`,
-        jsonlContent
-      );
+      await mode.writeFile(spansFilePath, jsonlContent);
 
       // Write metadata about the spans snapshot
+      const metadataFilePath = `/phoenix/projects/${projectName}/spans/metadata.json`;
+      logger.log(`Writing metadata to ${metadataFilePath}`);
       const metadata = {
         project: projectName,
         spanCount: spans.length,
@@ -133,10 +157,7 @@ export async function snapshotSpans(
         snapshotTime: new Date().toISOString(),
       };
 
-      await mode.writeFile(
-        `/phoenix/projects/${projectName}/spans/metadata.json`,
-        JSON.stringify(metadata, null, 2)
-      );
+      await mode.writeFile(metadataFilePath, JSON.stringify(metadata, null, 2));
     }, `fetching spans for project ${projectName}`);
   }
 }

package/src/snapshot/utils.ts

@@ -0,0 +1,140 @@
+/**
+ * Snapshot discovery utilities
+ *
+ * Functions for listing and finding snapshots in the local filesystem.
+ */
+
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+import * as os from "node:os";
+
+/**
+ * Information about a single snapshot
+ */
+export interface SnapshotInfo {
+  /** Absolute path to the snapshot directory (the 'phoenix' subdirectory) */
+  path: string;
+  /** Timestamp when the snapshot was created (from directory name) */
+  timestamp: Date;
+  /** Unique identifier for the snapshot (directory name) */
+  id: string;
+}
+
+/**
+ * Get the base snapshots directory path
+ */
+export function getSnapshotsDir(): string {
+  return path.join(os.homedir(), ".phoenix-insight", "snapshots");
+}
+
+/**
+ * Parse a snapshot directory name to extract timestamp
+ *
+ * Directory names are in format: `<timestamp>-<random>` where timestamp is Date.now()
+ * Example: "1704067200000-abc123" -> Date(2024-01-01T00:00:00.000Z)
+ *
+ * @param dirName - The directory name to parse
+ * @returns The parsed timestamp as Date, or null if invalid
+ */
+function parseSnapshotDirName(dirName: string): Date | null {
+  // Format: <timestamp>-<random>
+  const match = dirName.match(/^(\d+)-[\w]+$/);
+  if (!match || !match[1]) {
+    return null;
+  }
+
+  const timestamp = parseInt(match[1], 10);
+  if (isNaN(timestamp) || timestamp <= 0) {
+    return null;
+  }
+
+  const date = new Date(timestamp);
+  // Validate the date is reasonable (between year 2000 and year 3000)
+  // Use UTC year to avoid timezone issues
+  const year = date.getUTCFullYear();
+  if (year < 2000 || year > 3000) {
+    return null;
+  }
+
+  return date;
+}
+
+/**
+ * List all available snapshots
+ *
+ * Scans the snapshots directory and returns information about each valid snapshot.
+ * Results are sorted by timestamp descending (most recent first).
+ *
+ * @returns Array of snapshot info objects, sorted by timestamp descending
+ */
+export async function listSnapshots(): Promise<SnapshotInfo[]> {
+  const snapshotsDir = getSnapshotsDir();
+
+  // Check if snapshots directory exists
+  try {
+    await fs.access(snapshotsDir);
+  } catch {
+    // Directory doesn't exist - return empty array
+    return [];
+  }
+
+  // Read directory contents
+  let entries: string[];
+  try {
+    entries = await fs.readdir(snapshotsDir);
+  } catch {
+    // Cannot read directory - return empty array
+    return [];
+  }
+
+  // Filter and parse valid snapshot directories
+  const snapshots: SnapshotInfo[] = [];
+
+  for (const entry of entries) {
+    const timestamp = parseSnapshotDirName(entry);
+    if (!timestamp) {
+      // Invalid directory name format - skip
+      continue;
+    }
+
+    const snapshotPath = path.join(snapshotsDir, entry, "phoenix");
+
+    // Verify the phoenix subdirectory exists
+    try {
+      const stat = await fs.stat(snapshotPath);
+      if (!stat.isDirectory()) {
+        continue;
+      }
+    } catch {
+      // Phoenix subdirectory doesn't exist or can't be accessed - skip
+      continue;
+    }
+
+    snapshots.push({
+      path: snapshotPath,
+      timestamp,
+      id: entry,
+    });
+  }
+
+  // Sort by timestamp descending (most recent first)
+  snapshots.sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime());
+
+  return snapshots;
+}
+
+/**
+ * Get the latest (most recent) snapshot
+ *
+ * @returns The most recent snapshot info, or null if no snapshots exist
+ */
+export async function getLatestSnapshot(): Promise<SnapshotInfo | null> {
+  const snapshots = await listSnapshots();
+
+  if (snapshots.length === 0) {
+    return null;
+  }
+
+  // First element is the most recent due to descending sort
+  return snapshots[0] ?? null;
+}