@dreki-gg/pi-datadog 0.2.0 → 0.2.4

package/CHANGELOG.md

@@ -1,5 +1,37 @@
 # @dreki-gg/pi-datadog
 
+## 0.2.4
+
+### Patch Changes
+
+- Return a compact digest inline and write full log entries to a temp file.
+
+  `datadog_logs_search` no longer dumps truncated log entries inline. Instead it returns a token-light digest (result count, status breakdown, services, pagination) plus the path to a temp file containing the complete, untruncated results (full messages and attributes). The agent reads that file with the `read` tool when it needs actual log content — saving context tokens, removing the truncation blind spot, and avoiding extra rate-limited re-queries. Tool guidelines were updated to point at the file.
+
+## 0.2.3
+
+### Patch Changes
+
+- Write full untruncated log search results to a temp file the agent can read.
+
+  The inline `datadog_logs_search` output still truncates long messages and attributes for brevity, but the complete, untruncated results are now also written to a temp file and the path is included in the response. When the agent needs full message content (status codes, paths, stack traces), it can `read` that file instead of re-querying Datadog — avoiding both the truncation blind spot and extra rate-limited requests. Tool guidelines were updated to point at the file.
+
+## 0.2.2
+
+### Patch Changes
+
+- Handle Datadog rate limits gracefully instead of failing on 429s.
+
+  The log search client now enables the Datadog SDK's built-in retry (up to 4 attempts on 429/5xx), which honours the `x-ratelimit-reset` header and waits exactly the window Datadog asks for before retrying. When retries are exhausted, the tool returns a clear, actionable message (distinguishing rate-limit, auth, and other API errors) telling the agent to back off and keep queries narrow/batched rather than firing again immediately. Tool guidelines were updated to reinforce this batching behaviour.
+
+## 0.2.1
+
+### Patch Changes
+
+- Load project-root `.env` files so `DD_API_KEY` / `DD_APP_KEY` defined there are picked up.
+
+  Previously credentials were only read from the process environment, so keys placed in a project's `.env` (without exporting them in the shell) were never found and the extension reported missing credentials. The extension now loads `<cwd>/.env` via Node's built-in `process.loadEnvFile` at session start, before the tool runs, and in the `/datadog` status command. Shell-exported variables still take precedence over `.env` values.
+
 ## 0.2.0
 
 ### Minor Changes

package/extensions/datadog/client.ts

@@ -98,18 +98,102 @@ export function buildQuery(params: LogSearchParams, config: DatadogProjectConfig
   return parts.join(' ');
 }
 
+/** Max automatic retry attempts on 429 / 5xx responses. */
+const MAX_RETRIES = 4;
+
 function createApiInstance(credentials: DatadogCredentials, site: string): v2.LogsApi {
   const configuration = client.createConfiguration({
     authMethods: {
       apiKeyAuth: credentials.apiKey,
       appKeyAuth: credentials.appKey,
     },
+    // Transparently retry on 429 / 5xx. The SDK honours the `x-ratelimit-reset`
+    // header, so it waits exactly the window Datadog asks for before retrying
+    // (falling back to exponential backoff otherwise). This is what keeps the
+    // tool from "crying" about rate limits on bursty usage.
+    enableRetry: true,
+    maxRetries: MAX_RETRIES,
   });
   configuration.setServerVariables({ site });
 
   return new v2.LogsApi(configuration);
 }
 
+export interface DatadogErrorInfo {
+  /** Human-readable, agent-facing message. */
+  message: string;
+  /** HTTP status code, when the error came from the Datadog API. */
+  code?: number;
+  /** True when the error is a rate-limit (429) after retries were exhausted. */
+  isRateLimit: boolean;
+}
+
+/**
+ * Classifies an error thrown by the Datadog SDK into a clear, actionable message.
+ *
+ * The SDK throws an `ApiException`-shaped object (`{ code, body }`) on non-2xx
+ * responses. A 429 here means retries were already exhausted, so the message
+ * tells the agent to back off and keep queries narrow/batched rather than
+ * firing more requests.
+ */
+export function describeDatadogError(err: unknown): DatadogErrorInfo {
+  const code = extractStatusCode(err);
+  const detail = extractApiMessage(err);
+
+  if (code === 429) {
+    return {
+      code,
+      isRateLimit: true,
+      message:
+        `Rate limited by Datadog (429) after ${MAX_RETRIES} retries${detail ? `: ${detail}` : ''}. ` +
+        'Wait before retrying, and keep queries narrow/batched instead of firing many in quick succession.',
+    };
+  }
+
+  if (code === 401 || code === 403) {
+    return {
+      code,
+      isRateLimit: false,
+      message: `Datadog auth error (${code})${detail ? `: ${detail}` : ''}. Check DD_API_KEY / DD_APP_KEY.`,
+    };
+  }
+
+  if (code !== undefined) {
+    return {
+      code,
+      isRateLimit: false,
+      message: `Datadog API error (${code})${detail ? `: ${detail}` : ''}.`,
+    };
+  }
+
+  return {
+    isRateLimit: false,
+    message: `Datadog API error: ${err instanceof Error ? err.message : String(err)}`,
+  };
+}
+
+function extractStatusCode(err: unknown): number | undefined {
+  if (typeof err === 'object' && err !== null && 'code' in err) {
+    const code = (err as { code: unknown }).code;
+    if (typeof code === 'number') return code;
+  }
+  return undefined;
+}
+
+function extractApiMessage(err: unknown): string | undefined {
+  if (typeof err !== 'object' || err === null || !('body' in err)) return undefined;
+  const body = (err as { body: unknown }).body;
+
+  if (typeof body === 'string') return body || undefined;
+  if (typeof body === 'object' && body !== null && 'errors' in body) {
+    const errors = (body as { errors: unknown }).errors;
+    if (Array.isArray(errors) && errors.length > 0) {
+      return errors.map((e) => String(e)).join('; ');
+    }
+  }
+  return undefined;
+}
+
 function normalizeLogEntry(log: v2.Log): LogEntry {
   const attrs = log.attributes;
   return {

package/extensions/datadog/dotenv.ts

@@ -0,0 +1,85 @@
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+/**
+ * Loads a project-root `.env` file into `process.env`.
+ *
+ * Primary path uses Node's built-in `process.loadEnvFile` (Node >= 20.12).
+ * When that API is unavailable (older Node, or non-Node runtimes such as the
+ * Bun test runner) it falls back to a minimal inline parser with identical
+ * precedence semantics.
+ *
+ * Precedence follows the dotenv convention: variables already present in the
+ * environment (e.g. exported by the shell) take precedence over `.env` values;
+ * the file only fills in the gaps.
+ *
+ * Returns `true` if a `.env` file was found and loaded, `false` if none exists.
+ * A missing file or any read/parse error is swallowed so a malformed `.env`
+ * never breaks the extension.
+ */
+export function loadDotEnv(cwd: string): boolean {
+  const envPath = join(cwd, '.env');
+
+  if (typeof process.loadEnvFile === 'function') {
+    try {
+      process.loadEnvFile(envPath);
+      return true;
+    } catch (err: unknown) {
+      if (isENOENT(err)) return false;
+      return false;
+    }
+  }
+
+  return loadDotEnvFallback(envPath);
+}
+
+function loadDotEnvFallback(envPath: string): boolean {
+  let raw: string;
+  try {
+    raw = readFileSync(envPath, 'utf-8');
+  } catch (err: unknown) {
+    return isENOENT(err) ? false : false;
+  }
+
+  for (const parsed of parseDotEnv(raw)) {
+    // Existing env wins — only fill in unset keys.
+    if (process.env[parsed.key] === undefined) {
+      process.env[parsed.key] = parsed.value;
+    }
+  }
+  return true;
+}
+
+function parseDotEnv(raw: string): Array<{ key: string; value: string }> {
+  const result: Array<{ key: string; value: string }> = [];
+
+  for (const line of raw.split('\n')) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#')) continue;
+
+    const withoutExport = trimmed.startsWith('export ') ? trimmed.slice('export '.length) : trimmed;
+
+    const eq = withoutExport.indexOf('=');
+    if (eq === -1) continue;
+
+    const key = withoutExport.slice(0, eq).trim();
+    if (!key) continue;
+
+    let value = withoutExport.slice(eq + 1).trim();
+    // Strip matching surrounding quotes.
+    if (
+      (value.startsWith('"') && value.endsWith('"')) ||
+      (value.startsWith("'") && value.endsWith("'"))
+    ) {
+      value = value.slice(1, -1);
+    }
+
+    result.push({ key, value });
+  }
+
+  return result;
+}
+
+function isENOENT(err: unknown): boolean {
+  return err instanceof Error && 'code' in err && (err as NodeJS.ErrnoException).code === 'ENOENT';
+}

package/extensions/datadog/format.ts

@@ -3,8 +3,16 @@ import type { LogEntry, LogSearchResult } from './client.js';
 const MAX_MESSAGE_LENGTH = 500;
 const MAX_ATTRIBUTES_LENGTH = 300;
 
+export interface FormatOptions {
+  /** Max message length before truncation. Use Infinity to disable. */
+  maxMessageLength?: number;
+  /** Max attributes JSON length before truncation. Use Infinity to disable. */
+  maxAttributesLength?: number;
+}
+
 /**
  * Truncates a string to the given max length, appending "…" if truncated.
+ * A maxLength of Infinity disables truncation.
  */
 function truncate(text: string, maxLength: number): string {
   if (text.length <= maxLength) return text;
@@ -14,7 +22,7 @@ function truncate(text: string, maxLength: number): string {
 /**
  * Formats a single log entry as a compact markdown block.
  */
-function formatLogEntry(log: LogEntry, index: number): string {
+function formatLogEntry(log: LogEntry, index: number, opts: Required<FormatOptions>): string {
   const statusIcon = getStatusIcon(log.status);
   const header = `### ${index + 1}. ${statusIcon} \`${log.status}\` — ${log.timestamp}`;
 
@@ -24,12 +32,12 @@ function formatLogEntry(log: LogEntry, index: number): string {
   if (log.host !== 'unknown') lines.push(`**Host:** ${log.host}`);
 
   if (log.message) {
-    lines.push(`**Message:**\n\`\`\`\n${truncate(log.message, MAX_MESSAGE_LENGTH)}\n\`\`\``);
+    lines.push(`**Message:**\n\`\`\`\n${truncate(log.message, opts.maxMessageLength)}\n\`\`\``);
   }
 
   const attrKeys = Object.keys(log.attributes);
   if (attrKeys.length > 0) {
-    const attrStr = truncate(JSON.stringify(log.attributes, null, 2), MAX_ATTRIBUTES_LENGTH);
+    const attrStr = truncate(JSON.stringify(log.attributes, null, 2), opts.maxAttributesLength);
     lines.push(`**Attributes:**\n\`\`\`json\n${attrStr}\n\`\`\``);
   }
 
@@ -51,9 +59,17 @@ function getStatusIcon(status: string): string {
 }
 
 /**
- * Formats the full search result for LLM consumption.
+ * Formats the search result for LLM consumption.
+ *
+ * By default messages and attributes are truncated for a concise inline view.
+ * Pass `{ maxMessageLength: Infinity, maxAttributesLength: Infinity }` to render
+ * the complete, untruncated result (e.g. for writing to a temp file).
  */
-export function formatSearchResult(result: LogSearchResult): string {
+export function formatSearchResult(result: LogSearchResult, opts: FormatOptions = {}): string {
+  const resolved: Required<FormatOptions> = {
+    maxMessageLength: opts.maxMessageLength ?? MAX_MESSAGE_LENGTH,
+    maxAttributesLength: opts.maxAttributesLength ?? MAX_ATTRIBUTES_LENGTH,
+  };
   const lines: string[] = [];
 
   lines.push(`## Datadog Log Search Results`);
@@ -77,13 +93,50 @@ export function formatSearchResult(result: LogSearchResult): string {
   lines.push('');
 
   for (let i = 0; i < result.logs.length; i++) {
-    lines.push(formatLogEntry(result.logs[i], i));
+    lines.push(formatLogEntry(result.logs[i], i, resolved));
     if (i < result.logs.length - 1) lines.push('');
   }
 
   return lines.join('\n');
 }
 
+/**
+ * Builds a compact, token-light digest of a search result for inline tool
+ * output. The full per-entry detail lives in the temp file referenced by
+ * `resultsFile`; this just gives the agent enough to decide whether to read it.
+ */
+export function formatResultDigest(result: LogSearchResult, resultsFile?: string): string {
+  const lines: string[] = [];
+  lines.push(
+    `## Datadog Log Search — ${result.totalCount} log${result.totalCount === 1 ? '' : 's'}`,
+  );
+  lines.push(`**Query:** \`${result.query}\``);
+  lines.push(`**Time range:** ${result.from} → ${result.to}`);
+
+  if (result.logs.length === 0) {
+    lines.push('');
+    lines.push('No logs found matching the query.');
+    return lines.join('\n');
+  }
+
+  const summary = formatSearchSummary(result);
+  const breakdown = Object.entries(summary.statusBreakdown)
+    .map(([status, count]) => `${status}: ${count}`)
+    .join(', ');
+  if (breakdown) lines.push(`**Status:** ${breakdown}`);
+  if (summary.services.length > 0) lines.push(`**Services:** ${summary.services.join(', ')}`);
+  if (result.cursor) lines.push(`**Pagination:** more results available (cursor present)`);
+
+  if (resultsFile) {
+    lines.push('');
+    lines.push(
+      `📄 **Full results** (complete messages & attributes): \`${resultsFile}\`\nUse the \`read\` tool on this file to view every log entry.`,
+    );
+  }
+
+  return lines.join('\n');
+}
+
 /**
  * Formats a summary of the search result for tool details.
  */

package/extensions/datadog/index.ts

@@ -7,8 +7,10 @@ import {
   getCredentialStatus,
   loadProjectConfig,
 } from './config.js';
-import { searchLogs } from './client.js';
-import { formatSearchResult, formatSearchSummary } from './format.js';
+import { searchLogs, describeDatadogError } from './client.js';
+import { loadDotEnv } from './dotenv.js';
+import { formatSearchResult, formatSearchSummary, formatResultDigest } from './format.js';
+import { writeResultsFile } from './output.js';
 
 const SORT_ENUM = ['newest', 'oldest'] as const;
 
@@ -16,13 +18,18 @@ const TOOL_GUIDELINES = [
   'Use `datadog_logs_search` to search production logs in Datadog. It uses Datadog query syntax (e.g. `status:error`, `@http.status_code:500`, `service:my-api`).',
   '`datadog_logs_search` auto-applies project defaults for service and environment from `.pi/datadog.json` — only override when the user explicitly asks for a different service or env.',
   'When `datadog_logs_search` returns many results, summarize the patterns (error types, frequency, affected services) instead of listing every log entry.',
+  'The inline `datadog_logs_search` output is a compact digest (counts, status breakdown, services). The complete log entries — full messages and attributes — are written to a temp file whose path is in the response. To inspect actual log content (status codes, paths, stack traces), use the `read` tool on that file instead of re-querying.',
   'Use appropriate time ranges with `datadog_logs_search`: "15m" for recent issues, "1h" for general debugging, "24h" or "7d" for trend analysis.',
+  "Datadog is rate-limit sensitive: prefer a single narrow, well-scoped `datadog_logs_search` over many broad calls in quick succession. The tool already auto-retries on 429 (honouring Datadog's reset window) — if it still reports a rate limit, wait before retrying rather than firing again immediately.",
 ];
 
 export default function datadogExtension(pi: ExtensionAPI) {
   let projectConfig: DatadogProjectConfig | null = null;
 
   pi.on('session_start', async (_event, ctx) => {
+    // Load project-root .env so DD_API_KEY / DD_APP_KEY defined there are
+    // visible. Shell-exported vars still take precedence.
+    loadDotEnv(ctx.cwd);
     try {
       projectConfig = await loadProjectConfig(ctx.cwd);
     } catch (err) {
@@ -93,6 +100,9 @@ export default function datadogExtension(pi: ExtensionAPI) {
       _onUpdate?: unknown,
       ctx?: { cwd: string },
     ) {
+      // Ensure .env is loaded in case the tool runs before session_start.
+      if (ctx?.cwd) loadDotEnv(ctx.cwd);
+
       const credentials = getCredentials();
       if (!credentials) {
         const status = getCredentialStatus();
@@ -127,23 +137,42 @@ export default function datadogExtension(pi: ExtensionAPI) {
 
       try {
         const result = await searchLogs(params, config, credentials);
-        const formatted = formatSearchResult(result);
         const summary = formatSearchSummary(result);
 
+        // Write the full, untruncated results to a temp file the agent can read,
+        // and return only a compact digest inline to save tokens.
+        let resultsFile: string | undefined;
+        if (result.logs.length > 0) {
+          try {
+            const fullContent = formatSearchResult(result, {
+              maxMessageLength: Infinity,
+              maxAttributesLength: Infinity,
+            });
+            resultsFile = await writeResultsFile(fullContent);
+          } catch {
+            // Non-fatal: digest still carries the summary.
+            resultsFile = undefined;
+          }
+        }
+
         return {
-          content: [{ type: 'text' as const, text: formatted }],
-          details: summary as Record<string, unknown>,
+          content: [{ type: 'text' as const, text: formatResultDigest(result, resultsFile) }],
+          details: { ...summary, resultsFile } as Record<string, unknown>,
         };
       } catch (err) {
-        const message = err instanceof Error ? err.message : String(err);
+        const info = describeDatadogError(err);
         return {
           content: [
             {
               type: 'text' as const,
-              text: `❌ Datadog API error: ${message}`,
+              text: `❌ ${info.message}`,
             },
           ],
-          details: { error: 'api_error', message } as Record<string, unknown>,
+          details: {
+            error: info.isRateLimit ? 'rate_limited' : 'api_error',
+            code: info.code,
+            message: info.message,
+          } as Record<string, unknown>,
           isError: true,
         };
       }
@@ -160,6 +189,7 @@ export default function datadogExtension(pi: ExtensionAPI) {
         ui: { notify(message: string, level: 'info' | 'warning' | 'error'): void };
       },
     ) => {
+      loadDotEnv(ctx.cwd);
       const credStatus = getCredentialStatus();
       const config = projectConfig ?? (await loadProjectConfig(ctx.cwd).catch(() => null));
 

package/extensions/datadog/output.ts

@@ -0,0 +1,17 @@
+import { mkdtemp, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+/**
+ * Writes the full (untruncated) search results to a temp file the agent can
+ * read freely with the `read` tool, sidestepping the inline truncation that
+ * hides long log messages and attributes.
+ *
+ * Returns the absolute path to the written file.
+ */
+export async function writeResultsFile(content: string): Promise<string> {
+  const dir = await mkdtemp(join(tmpdir(), 'pi-datadog-'));
+  const path = join(dir, `logs-${Date.now()}.md`);
+  await writeFile(path, content, 'utf-8');
+  return path;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dreki-gg/pi-datadog",
-  "version": "0.2.0",
+  "version": "0.2.4",
   "description": "Datadog log search tools for pi — query production logs with project-aware context",
   "keywords": [
     "pi-package",