@poncho-ai/harness 0.28.3 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +5 -5
- package/.turbo/turbo-lint.log +6 -0
- package/.turbo/turbo-test.log +34 -0
- package/CHANGELOG.md +11 -0
- package/dist/index.d.ts +23 -5
- package/dist/index.js +225 -81
- package/package.json +4 -2
- package/src/config.ts +4 -1
- package/src/harness.ts +135 -11
- package/src/state.ts +5 -0
- package/src/telemetry.ts +20 -10
package/.turbo/turbo-build.log
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
|
|
2
|
-
> @poncho-ai/harness@0.
|
|
2
|
+
> @poncho-ai/harness@0.29.0 build /Users/cesar/Dev/latitude/poncho-ai/packages/harness
|
|
3
3
|
> node scripts/embed-docs.js && tsup src/index.ts --format esm --dts
|
|
4
4
|
|
|
5
5
|
[embed-docs] Generated poncho-docs.ts with 4 topics
|
|
@@ -8,8 +8,8 @@
|
|
|
8
8
|
[34mCLI[39m tsup v8.5.1
|
|
9
9
|
[34mCLI[39m Target: es2022
|
|
10
10
|
[34mESM[39m Build start
|
|
11
|
-
[32mESM[39m [1mdist/index.js [22m[
|
|
12
|
-
[32mESM[39m ⚡️ Build success in
|
|
11
|
+
[32mESM[39m [1mdist/index.js [22m[32m297.56 KB[39m
|
|
12
|
+
[32mESM[39m ⚡️ Build success in 32ms
|
|
13
13
|
[34mDTS[39m Build start
|
|
14
|
-
[32mDTS[39m ⚡️ Build success in
|
|
15
|
-
[32mDTS[39m [1mdist/index.d.ts [22m[
|
|
14
|
+
[32mDTS[39m ⚡️ Build success in 4608ms
|
|
15
|
+
[32mDTS[39m [1mdist/index.d.ts [22m[32m30.41 KB[39m
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
|
|
2
|
+
> @poncho-ai/harness@0.26.0 test /Users/cesar/Dev/latitude/poncho-ai/packages/harness
|
|
3
|
+
> vitest
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
[7m[1m[36m RUN [39m[22m[27m [36mv1.6.1[39m [90m/Users/cesar/Dev/latitude/poncho-ai/packages/harness[39m
|
|
7
|
+
|
|
8
|
+
[90mstdout[2m | test/mcp.test.ts[2m > [22m[2mmcp bridge protocol transports[2m > [22m[2mdiscovers and calls tools over streamable HTTP[22m[39m
|
|
9
|
+
[poncho][mcp] {"event":"catalog.loaded","server":"remote","discoveredCount":1}
|
|
10
|
+
[poncho][mcp] {"event":"tools.selected","requestedPatternCount":1,"registeredCount":1,"filteredByPolicyCount":0,"filteredByIntentCount":0}
|
|
11
|
+
|
|
12
|
+
[90mstdout[2m | test/mcp.test.ts[2m > [22m[2mmcp bridge protocol transports[2m > [22m[2msends custom headers alongside bearer token[22m[39m
|
|
13
|
+
[poncho][mcp] {"event":"catalog.loaded","server":"custom-headers","discoveredCount":1}
|
|
14
|
+
|
|
15
|
+
[90mstderr[2m | test/mcp.test.ts[2m > [22m[2mmcp bridge protocol transports[2m > [22m[2mskips discovery when bearer token env value is missing[22m[39m
|
|
16
|
+
[90mstdout[2m | test/mcp.test.ts[2m > [22m[2mmcp bridge protocol transports[2m > [22m[2mselects discovered tools by requested patterns[22m[39m
|
|
17
|
+
[poncho][mcp] {"event":"auth.token_missing","server":"remote","tokenEnv":"MISSING_TOKEN_ENV"}
|
|
18
|
+
[poncho][mcp] {"event":"catalog.loaded","server":"remote","discoveredCount":2}
|
|
19
|
+
[poncho][mcp] {"event":"tools.selected","requestedPatternCount":1,"registeredCount":1,"filteredByPolicyCount":0,"filteredByIntentCount":1}
|
|
20
|
+
|
|
21
|
+
[poncho][mcp] {"event":"tools.selected","requestedPatternCount":1,"registeredCount":2,"filteredByPolicyCount":0,"filteredByIntentCount":0}
|
|
22
|
+
|
|
23
|
+
[90mstdout[2m | test/mcp.test.ts[2m > [22m[2mmcp bridge protocol transports[2m > [22m[2mskips discovery when bearer token env value is missing[22m[39m
|
|
24
|
+
[poncho][mcp] {"event":"tools.selected","requestedPatternCount":1,"registeredCount":0,"filteredByPolicyCount":0,"filteredByIntentCount":0}
|
|
25
|
+
|
|
26
|
+
[event] step:completed {"type":"step:completed","step":1,"duration":1}
|
|
27
|
+
[32m✓[39m test/telemetry.test.ts [2m ([22m[2m3 tests[22m[2m)[22m[90m 5[2mms[22m[39m
|
|
28
|
+
[event] step:started {"type":"step:started","step":2}
|
|
29
|
+
[32m✓[39m test/schema-converter.test.ts [2m ([22m[2m27 tests[22m[2m)[22m[90m 13[2mms[22m[39m
|
|
30
|
+
[90mstdout[2m | test/mcp.test.ts[2m > [22m[2mmcp bridge protocol transports[2m > [22m[2mreturns actionable errors for 403 permission failures[22m[39m
|
|
31
|
+
[poncho][mcp] {"event":"catalog.loaded","server":"remote","discoveredCount":1}
|
|
32
|
+
[poncho][mcp] {"event":"tools.selected","requestedPatternCount":1,"registeredCount":1,"filteredByPolicyCount":0,"filteredByIntentCount":0}
|
|
33
|
+
|
|
34
|
+
[32m✓[39m test/mcp.test.ts [2m ([22m[2m7 tests[22m[2m)[22m[90m 84[2mms[22m[39m
|
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,16 @@
|
|
|
1
1
|
# @poncho-ai/harness
|
|
2
2
|
|
|
3
|
+
## 0.29.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- [#51](https://github.com/cesr/poncho-ai/pull/51) [`eb661a5`](https://github.com/cesr/poncho-ai/commit/eb661a554da6839702651671db8a8820ceb13f35) Thanks [@cesr](https://github.com/cesr)! - Add generic OTLP trace exporter for sending OpenTelemetry traces to any collector (Jaeger, Grafana Tempo, Honeycomb, etc.). Configure via `telemetry.otlp` as a URL string or `{ url, headers }` object. Works alongside or instead of Latitude telemetry.
|
|
8
|
+
|
|
9
|
+
### Patch Changes
|
|
10
|
+
|
|
11
|
+
- Updated dependencies [[`eb661a5`](https://github.com/cesr/poncho-ai/commit/eb661a554da6839702651671db8a8820ceb13f35)]:
|
|
12
|
+
- @poncho-ai/sdk@1.6.2
|
|
13
|
+
|
|
3
14
|
## 0.28.3
|
|
4
15
|
|
|
5
16
|
### Patch Changes
|
package/dist/index.d.ts
CHANGED
|
@@ -176,6 +176,11 @@ interface Conversation {
|
|
|
176
176
|
/** Harness-internal message chain preserved across continuation runs.
|
|
177
177
|
* Cleared when a run completes without continuation. */
|
|
178
178
|
_continuationMessages?: Message[];
|
|
179
|
+
/** Full structured message chain from the last harness run, including
|
|
180
|
+
* tool-call and tool-result messages the model needs for context.
|
|
181
|
+
* Unlike `_continuationMessages`, this is always set after a run
|
|
182
|
+
* and does NOT signal that a continuation is pending. */
|
|
183
|
+
_harnessMessages?: Message[];
|
|
179
184
|
createdAt: number;
|
|
180
185
|
updatedAt: number;
|
|
181
186
|
}
|
|
@@ -411,7 +416,10 @@ interface PonchoConfig extends McpConfig {
|
|
|
411
416
|
};
|
|
412
417
|
telemetry?: {
|
|
413
418
|
enabled?: boolean;
|
|
414
|
-
otlp?: string
|
|
419
|
+
otlp?: string | {
|
|
420
|
+
url: string;
|
|
421
|
+
headers?: Record<string, string>;
|
|
422
|
+
};
|
|
415
423
|
latitude?: {
|
|
416
424
|
apiKeyEnv?: string;
|
|
417
425
|
projectIdEnv?: string;
|
|
@@ -623,6 +631,9 @@ declare class AgentHarness {
|
|
|
623
631
|
private readonly activeSkillNames;
|
|
624
632
|
private readonly registeredMcpToolNames;
|
|
625
633
|
private latitudeTelemetry?;
|
|
634
|
+
private otlpSpanProcessor?;
|
|
635
|
+
private otlpTracerProvider?;
|
|
636
|
+
private hasOtlpExporter;
|
|
626
637
|
private insideTelemetryCapture;
|
|
627
638
|
private _browserSession?;
|
|
628
639
|
private _browserMod?;
|
|
@@ -686,8 +697,9 @@ declare class AgentHarness {
|
|
|
686
697
|
shutdown(): Promise<void>;
|
|
687
698
|
listTools(): ToolDefinition[];
|
|
688
699
|
/**
|
|
689
|
-
* Wraps the run() generator with
|
|
690
|
-
*
|
|
700
|
+
* Wraps the run() generator with telemetry capture for complete trace coverage.
|
|
701
|
+
* Supports Latitude, generic OTLP, or both simultaneously.
|
|
702
|
+
* Streams events in real-time using an event queue pattern.
|
|
691
703
|
*/
|
|
692
704
|
runWithTelemetry(input: RunInput): AsyncGenerator<AgentEvent>;
|
|
693
705
|
compact(messages: Message[], options?: CompactMessagesOptions): Promise<CompactResult>;
|
|
@@ -804,9 +816,15 @@ declare const createSkillTools: (skills: SkillMetadata[], options?: {
|
|
|
804
816
|
}) => ToolDefinition[];
|
|
805
817
|
declare const normalizeScriptPolicyPath: (relativePath: string) => string;
|
|
806
818
|
|
|
819
|
+
interface OtlpConfig {
|
|
820
|
+
url: string;
|
|
821
|
+
headers?: Record<string, string>;
|
|
822
|
+
}
|
|
823
|
+
type OtlpOption = string | OtlpConfig;
|
|
824
|
+
declare function normalizeOtlp(opt: OtlpOption | undefined): OtlpConfig | undefined;
|
|
807
825
|
interface TelemetryConfig {
|
|
808
826
|
enabled?: boolean;
|
|
809
|
-
otlp?:
|
|
827
|
+
otlp?: OtlpOption;
|
|
810
828
|
latitude?: {
|
|
811
829
|
apiKeyEnv?: string;
|
|
812
830
|
projectIdEnv?: string;
|
|
@@ -824,4 +842,4 @@ declare class TelemetryEmitter {
|
|
|
824
842
|
|
|
825
843
|
declare const createSubagentTools: (manager: SubagentManager) => ToolDefinition[];
|
|
826
844
|
|
|
827
|
-
export { type AgentFrontmatter, AgentHarness, type AgentIdentity, type AgentLimitsConfig, type AgentModelConfig, type BuiltInToolToggles, type CompactMessagesOptions, type CompactResult, type CompactionConfig, type Conversation, type ConversationState, type ConversationStore, type ConversationSummary, type CronJobConfig, type HarnessOptions, type HarnessRunOutput, InMemoryConversationStore, InMemoryStateStore, LatitudeCapture, type LatitudeCaptureConfig, LocalMcpBridge, LocalUploadStore, type MainMemory, type McpConfig, type MemoryConfig, type MemoryStore, type MessagingChannelConfig, type ModelProviderFactory, PONCHO_UPLOAD_SCHEME, type ParsedAgent, type PendingSubagentResult, type PonchoConfig, type ProviderConfig, type RemoteMcpServerConfig, type RuntimeRenderContext, S3UploadStore, STORAGE_SCHEMA_VERSION, type SkillContextEntry, type SkillMetadata, type StateConfig, type StateProviderName, type StateStore, type StorageConfig, type SubagentManager, type SubagentResult, type SubagentSpawnResult, type SubagentSummary, type TelemetryConfig, TelemetryEmitter, type ToolAccess, type ToolCall, ToolDispatcher, type ToolExecutionResult, type UploadStore, type UploadsConfig, VercelBlobUploadStore, buildAgentDirectoryName, buildSkillContextWindow, compactMessages, createConversationStore, createDefaultTools, createDeleteDirectoryTool, createDeleteTool, createEditTool, createMemoryStore, createMemoryTools, createModelProvider, createSearchTools, createSkillTools, createStateStore, createSubagentTools, createUploadStore, createWriteTool, deriveUploadKey, ensureAgentIdentity, estimateTokens, estimateTotalTokens, findSafeSplitPoint, generateAgentId, getAgentStoreDirectory, getModelContextWindow, getPonchoStoreRoot, jsonSchemaToZod, loadPonchoConfig, loadSkillContext, loadSkillInstructions, loadSkillMetadata, normalizeScriptPolicyPath, parseAgentFile, parseAgentMarkdown, ponchoDocsTool, readSkillResource, renderAgentPrompt, resolveAgentIdentity, resolveCompactionConfig, resolveMemoryConfig, resolveSkillDirs, resolveStateConfig, slugifyStorageComponent };
|
|
845
|
+
export { type AgentFrontmatter, AgentHarness, type AgentIdentity, type AgentLimitsConfig, type AgentModelConfig, type BuiltInToolToggles, type CompactMessagesOptions, type CompactResult, type CompactionConfig, type Conversation, type ConversationState, type ConversationStore, type ConversationSummary, type CronJobConfig, type HarnessOptions, type HarnessRunOutput, InMemoryConversationStore, InMemoryStateStore, LatitudeCapture, type LatitudeCaptureConfig, LocalMcpBridge, LocalUploadStore, type MainMemory, type McpConfig, type MemoryConfig, type MemoryStore, type MessagingChannelConfig, type ModelProviderFactory, type OtlpConfig, type OtlpOption, PONCHO_UPLOAD_SCHEME, type ParsedAgent, type PendingSubagentResult, type PonchoConfig, type ProviderConfig, type RemoteMcpServerConfig, type RuntimeRenderContext, S3UploadStore, STORAGE_SCHEMA_VERSION, type SkillContextEntry, type SkillMetadata, type StateConfig, type StateProviderName, type StateStore, type StorageConfig, type SubagentManager, type SubagentResult, type SubagentSpawnResult, type SubagentSummary, type TelemetryConfig, TelemetryEmitter, type ToolAccess, type ToolCall, ToolDispatcher, type ToolExecutionResult, type UploadStore, type UploadsConfig, VercelBlobUploadStore, buildAgentDirectoryName, buildSkillContextWindow, compactMessages, createConversationStore, createDefaultTools, createDeleteDirectoryTool, createDeleteTool, createEditTool, createMemoryStore, createMemoryTools, createModelProvider, createSearchTools, createSkillTools, createStateStore, createSubagentTools, createUploadStore, createWriteTool, deriveUploadKey, ensureAgentIdentity, estimateTokens, estimateTotalTokens, findSafeSplitPoint, generateAgentId, getAgentStoreDirectory, getModelContextWindow, getPonchoStoreRoot, jsonSchemaToZod, loadPonchoConfig, loadSkillContext, loadSkillInstructions, loadSkillMetadata, normalizeOtlp, normalizeScriptPolicyPath, parseAgentFile, parseAgentMarkdown, ponchoDocsTool, readSkillResource, renderAgentPrompt, resolveAgentIdentity, resolveCompactionConfig, resolveMemoryConfig, resolveSkillDirs, resolveStateConfig, slugifyStorageComponent };
|
package/dist/index.js
CHANGED
|
@@ -1529,11 +1529,17 @@ export default {
|
|
|
1529
1529
|
},
|
|
1530
1530
|
},
|
|
1531
1531
|
|
|
1532
|
-
// Telemetry destination
|
|
1532
|
+
// Telemetry destination \u2014 generic OTLP and/or Latitude
|
|
1533
1533
|
telemetry: {
|
|
1534
1534
|
enabled: true,
|
|
1535
|
+
// Generic OTLP: string shorthand or { url, headers? } object
|
|
1535
1536
|
otlp: process.env.OTEL_EXPORTER_OTLP_ENDPOINT,
|
|
1536
|
-
//
|
|
1537
|
+
// With auth headers (Honeycomb, Grafana Cloud, etc.):
|
|
1538
|
+
// otlp: {
|
|
1539
|
+
// url: 'https://api.honeycomb.io/v1/traces',
|
|
1540
|
+
// headers: { 'x-honeycomb-team': process.env.HONEYCOMB_API_KEY },
|
|
1541
|
+
// },
|
|
1542
|
+
// Latitude (reads from LATITUDE_API_KEY and LATITUDE_PROJECT_ID env vars by default)
|
|
1537
1543
|
latitude: {
|
|
1538
1544
|
// apiKeyEnv: 'LATITUDE_API_KEY', // default
|
|
1539
1545
|
// projectIdEnv: 'LATITUDE_PROJECT_ID', // default
|
|
@@ -1606,7 +1612,7 @@ Remote storage keys are namespaced and versioned, for example \`poncho:v1:<agent
|
|
|
1606
1612
|
| \`PONCHO_AUTH_TOKEN\` | No | Unified auth token (Web UI passphrase + API Bearer token) |
|
|
1607
1613
|
| \`PONCHO_INTERNAL_SECRET\` | No | Shared secret used by internal serverless callbacks (recommended for Vercel/Lambda) |
|
|
1608
1614
|
| \`PONCHO_SELF_BASE_URL\` | No | Explicit base URL for internal self-callbacks when auto-detection is unavailable |
|
|
1609
|
-
| \`OTEL_EXPORTER_OTLP_ENDPOINT\` | No |
|
|
1615
|
+
| \`OTEL_EXPORTER_OTLP_ENDPOINT\` | No | OTLP trace endpoint (Jaeger, Tempo, Honeycomb, etc.) |
|
|
1610
1616
|
| \`LATITUDE_API_KEY\` | No | Latitude dashboard integration |
|
|
1611
1617
|
| \`LATITUDE_PROJECT_ID\` | No | Latitude project identifier for capture traces |
|
|
1612
1618
|
| \`LATITUDE_PATH\` | No | Latitude prompt path for grouping traces |
|
|
@@ -1641,23 +1647,45 @@ Logs print to console:
|
|
|
1641
1647
|
[event] run:completed {"type":"run:completed","runId":"run_abc123","result":{"status":"completed","response":"...","steps":3,"tokens":{"input":1500,"output":840}}}
|
|
1642
1648
|
\`\`\`
|
|
1643
1649
|
|
|
1644
|
-
### Production telemetry
|
|
1650
|
+
### Production telemetry (generic OTLP)
|
|
1645
1651
|
|
|
1646
|
-
Send
|
|
1652
|
+
Send full OpenTelemetry traces (agent runs, LLM calls, tool executions) to any
|
|
1653
|
+
OTLP-compatible collector \u2014 Jaeger, Grafana Tempo, Honeycomb, Datadog, etc.
|
|
1647
1654
|
|
|
1648
1655
|
\`\`\`bash
|
|
1649
|
-
#
|
|
1650
|
-
OTEL_EXPORTER_OTLP_ENDPOINT=https://otel.example.com
|
|
1656
|
+
# Simple: just a URL
|
|
1657
|
+
OTEL_EXPORTER_OTLP_ENDPOINT=https://otel.example.com/v1/traces
|
|
1651
1658
|
\`\`\`
|
|
1652
1659
|
|
|
1653
|
-
|
|
1660
|
+
\`\`\`javascript
|
|
1661
|
+
// poncho.config.js \u2014 string shorthand
|
|
1662
|
+
export default {
|
|
1663
|
+
telemetry: {
|
|
1664
|
+
otlp: 'https://otel.example.com/v1/traces',
|
|
1665
|
+
}
|
|
1666
|
+
}
|
|
1667
|
+
\`\`\`
|
|
1668
|
+
|
|
1669
|
+
\`\`\`javascript
|
|
1670
|
+
// poncho.config.js \u2014 with auth headers (Honeycomb, Grafana Cloud, etc.)
|
|
1671
|
+
export default {
|
|
1672
|
+
telemetry: {
|
|
1673
|
+
otlp: {
|
|
1674
|
+
url: 'https://api.honeycomb.io/v1/traces',
|
|
1675
|
+
headers: {
|
|
1676
|
+
'x-honeycomb-team': process.env.HONEYCOMB_API_KEY,
|
|
1677
|
+
},
|
|
1678
|
+
},
|
|
1679
|
+
}
|
|
1680
|
+
}
|
|
1681
|
+
\`\`\`
|
|
1682
|
+
|
|
1683
|
+
You can also use a custom event handler for non-OTLP destinations:
|
|
1654
1684
|
|
|
1655
1685
|
\`\`\`javascript
|
|
1656
1686
|
// poncho.config.js
|
|
1657
1687
|
export default {
|
|
1658
1688
|
telemetry: {
|
|
1659
|
-
otlp: 'https://otel.example.com',
|
|
1660
|
-
// Or custom handler
|
|
1661
1689
|
handler: async (event) => {
|
|
1662
1690
|
await sendToMyLoggingService(event)
|
|
1663
1691
|
}
|
|
@@ -1687,6 +1715,8 @@ telemetry: {
|
|
|
1687
1715
|
}
|
|
1688
1716
|
\`\`\`
|
|
1689
1717
|
|
|
1718
|
+
Both \`otlp\` and \`latitude\` can be configured simultaneously \u2014 all spans flow to both destinations.
|
|
1719
|
+
|
|
1690
1720
|
## Security
|
|
1691
1721
|
|
|
1692
1722
|
### Protect your endpoint
|
|
@@ -4581,6 +4611,78 @@ var createSubagentTools = (manager) => [
|
|
|
4581
4611
|
|
|
4582
4612
|
// src/harness.ts
|
|
4583
4613
|
import { LatitudeTelemetry } from "@latitude-data/telemetry";
|
|
4614
|
+
import { trace, context as otelContext, SpanStatusCode } from "@opentelemetry/api";
|
|
4615
|
+
import { NodeTracerProvider, BatchSpanProcessor } from "@opentelemetry/sdk-trace-node";
|
|
4616
|
+
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
|
|
4617
|
+
|
|
4618
|
+
// src/telemetry.ts
|
|
4619
|
+
var MAX_FIELD_LENGTH = 200;
|
|
4620
|
+
function sanitizeEventForLog(event) {
|
|
4621
|
+
return JSON.stringify(event, (_key, value) => {
|
|
4622
|
+
if (typeof value === "string" && value.length > MAX_FIELD_LENGTH) {
|
|
4623
|
+
return `${value.slice(0, 80)}...[${value.length} chars]`;
|
|
4624
|
+
}
|
|
4625
|
+
return value;
|
|
4626
|
+
});
|
|
4627
|
+
}
|
|
4628
|
+
function normalizeOtlp(opt) {
|
|
4629
|
+
if (!opt) return void 0;
|
|
4630
|
+
if (typeof opt === "string") return opt ? { url: opt } : void 0;
|
|
4631
|
+
return opt.url ? opt : void 0;
|
|
4632
|
+
}
|
|
4633
|
+
var TelemetryEmitter = class {
|
|
4634
|
+
config;
|
|
4635
|
+
constructor(config) {
|
|
4636
|
+
this.config = config;
|
|
4637
|
+
}
|
|
4638
|
+
async emit(event) {
|
|
4639
|
+
if (this.config?.enabled === false) {
|
|
4640
|
+
return;
|
|
4641
|
+
}
|
|
4642
|
+
if (this.config?.handler) {
|
|
4643
|
+
await this.config.handler(event);
|
|
4644
|
+
return;
|
|
4645
|
+
}
|
|
4646
|
+
const otlp = normalizeOtlp(this.config?.otlp);
|
|
4647
|
+
if (otlp) {
|
|
4648
|
+
await this.sendOtlp(event, otlp);
|
|
4649
|
+
}
|
|
4650
|
+
process.stdout.write(`[event] ${event.type} ${sanitizeEventForLog(event)}
|
|
4651
|
+
`);
|
|
4652
|
+
}
|
|
4653
|
+
async sendOtlp(event, otlp) {
|
|
4654
|
+
try {
|
|
4655
|
+
await fetch(otlp.url, {
|
|
4656
|
+
method: "POST",
|
|
4657
|
+
headers: { "Content-Type": "application/json", ...otlp.headers },
|
|
4658
|
+
body: JSON.stringify({
|
|
4659
|
+
resourceLogs: [
|
|
4660
|
+
{
|
|
4661
|
+
scopeLogs: [
|
|
4662
|
+
{
|
|
4663
|
+
logRecords: [
|
|
4664
|
+
{
|
|
4665
|
+
timeUnixNano: String(Date.now() * 1e6),
|
|
4666
|
+
severityText: "INFO",
|
|
4667
|
+
body: { stringValue: event.type },
|
|
4668
|
+
attributes: [
|
|
4669
|
+
{
|
|
4670
|
+
key: "event.payload",
|
|
4671
|
+
value: { stringValue: JSON.stringify(event) }
|
|
4672
|
+
}
|
|
4673
|
+
]
|
|
4674
|
+
}
|
|
4675
|
+
]
|
|
4676
|
+
}
|
|
4677
|
+
]
|
|
4678
|
+
}
|
|
4679
|
+
]
|
|
4680
|
+
})
|
|
4681
|
+
});
|
|
4682
|
+
} catch {
|
|
4683
|
+
}
|
|
4684
|
+
}
|
|
4685
|
+
};
|
|
4584
4686
|
|
|
4585
4687
|
// src/tool-dispatcher.ts
|
|
4586
4688
|
var ToolDispatcher = class {
|
|
@@ -5113,6 +5215,9 @@ var AgentHarness = class _AgentHarness {
|
|
|
5113
5215
|
activeSkillNames = /* @__PURE__ */ new Set();
|
|
5114
5216
|
registeredMcpToolNames = /* @__PURE__ */ new Set();
|
|
5115
5217
|
latitudeTelemetry;
|
|
5218
|
+
otlpSpanProcessor;
|
|
5219
|
+
otlpTracerProvider;
|
|
5220
|
+
hasOtlpExporter = false;
|
|
5116
5221
|
insideTelemetryCapture = false;
|
|
5117
5222
|
_browserSession;
|
|
5118
5223
|
_browserMod;
|
|
@@ -5560,6 +5665,31 @@ var AgentHarness = class _AgentHarness {
|
|
|
5560
5665
|
`[poncho][telemetry] Latitude telemetry is configured but missing: ${missing.join(", ")}. Traces will NOT be sent.`
|
|
5561
5666
|
);
|
|
5562
5667
|
}
|
|
5668
|
+
const otlpConfig = telemetryEnabled ? normalizeOtlp(config?.telemetry?.otlp) : void 0;
|
|
5669
|
+
if (otlpConfig) {
|
|
5670
|
+
const exporter = new OTLPTraceExporter({
|
|
5671
|
+
url: otlpConfig.url,
|
|
5672
|
+
headers: otlpConfig.headers
|
|
5673
|
+
});
|
|
5674
|
+
const processor = new BatchSpanProcessor(exporter);
|
|
5675
|
+
this.otlpSpanProcessor = processor;
|
|
5676
|
+
if (this.latitudeTelemetry) {
|
|
5677
|
+
const globalProvider = trace.getTracerProvider();
|
|
5678
|
+
const delegate = globalProvider.getDelegate?.() ?? globalProvider;
|
|
5679
|
+
if (typeof delegate.addSpanProcessor === "function") {
|
|
5680
|
+
delegate.addSpanProcessor(processor);
|
|
5681
|
+
}
|
|
5682
|
+
console.info(`[poncho][telemetry] OTLP exporter added (piggybacking on Latitude provider) \u2192 ${otlpConfig.url}`);
|
|
5683
|
+
} else {
|
|
5684
|
+
const provider2 = new NodeTracerProvider({
|
|
5685
|
+
spanProcessors: [processor]
|
|
5686
|
+
});
|
|
5687
|
+
provider2.register();
|
|
5688
|
+
this.otlpTracerProvider = provider2;
|
|
5689
|
+
console.info(`[poncho][telemetry] OTLP exporter active (standalone provider) \u2192 ${otlpConfig.url}`);
|
|
5690
|
+
}
|
|
5691
|
+
this.hasOtlpExporter = true;
|
|
5692
|
+
}
|
|
5563
5693
|
}
|
|
5564
5694
|
async buildBrowserStoragePersistence(config, sessionId) {
|
|
5565
5695
|
const provider = config.storage?.provider ?? config.state?.provider ?? "local";
|
|
@@ -5710,13 +5840,31 @@ var AgentHarness = class _AgentHarness {
|
|
|
5710
5840
|
});
|
|
5711
5841
|
this.latitudeTelemetry = void 0;
|
|
5712
5842
|
}
|
|
5843
|
+
if (this.otlpSpanProcessor) {
|
|
5844
|
+
await this.otlpSpanProcessor.shutdown().catch((err) => {
|
|
5845
|
+
console.warn(
|
|
5846
|
+
`[poncho][telemetry] OTLP span processor shutdown error: ${err instanceof Error ? err.message : String(err)}`
|
|
5847
|
+
);
|
|
5848
|
+
});
|
|
5849
|
+
this.otlpSpanProcessor = void 0;
|
|
5850
|
+
}
|
|
5851
|
+
if (this.otlpTracerProvider) {
|
|
5852
|
+
await this.otlpTracerProvider.shutdown().catch((err) => {
|
|
5853
|
+
console.warn(
|
|
5854
|
+
`[poncho][telemetry] OTLP tracer provider shutdown error: ${err instanceof Error ? err.message : String(err)}`
|
|
5855
|
+
);
|
|
5856
|
+
});
|
|
5857
|
+
this.otlpTracerProvider = void 0;
|
|
5858
|
+
}
|
|
5859
|
+
this.hasOtlpExporter = false;
|
|
5713
5860
|
}
|
|
5714
5861
|
listTools() {
|
|
5715
5862
|
return this.dispatcher.list();
|
|
5716
5863
|
}
|
|
5717
5864
|
/**
|
|
5718
|
-
* Wraps the run() generator with
|
|
5719
|
-
*
|
|
5865
|
+
* Wraps the run() generator with telemetry capture for complete trace coverage.
|
|
5866
|
+
* Supports Latitude, generic OTLP, or both simultaneously.
|
|
5867
|
+
* Streams events in real-time using an event queue pattern.
|
|
5720
5868
|
*/
|
|
5721
5869
|
async *runWithTelemetry(input) {
|
|
5722
5870
|
const config = this.loadedConfig;
|
|
@@ -5783,6 +5931,39 @@ var AgentHarness = class _AgentHarness {
|
|
|
5783
5931
|
}
|
|
5784
5932
|
}
|
|
5785
5933
|
}
|
|
5934
|
+
} else if (this.hasOtlpExporter) {
|
|
5935
|
+
const tracer = trace.getTracer("poncho");
|
|
5936
|
+
const agentName = this.parsedAgent?.frontmatter.name ?? "agent";
|
|
5937
|
+
const rootSpan = tracer.startSpan(`agent.run ${agentName}`);
|
|
5938
|
+
rootSpan.setAttribute("poncho.agent.name", agentName);
|
|
5939
|
+
if (input.conversationId) {
|
|
5940
|
+
rootSpan.setAttribute("poncho.conversation.id", input.conversationId);
|
|
5941
|
+
}
|
|
5942
|
+
const spanContext = trace.setSpan(otelContext.active(), rootSpan);
|
|
5943
|
+
this.insideTelemetryCapture = true;
|
|
5944
|
+
try {
|
|
5945
|
+
const gen = this.run(input);
|
|
5946
|
+
let next;
|
|
5947
|
+
do {
|
|
5948
|
+
next = await otelContext.with(spanContext, () => gen.next());
|
|
5949
|
+
if (!next.done) yield next.value;
|
|
5950
|
+
} while (!next.done);
|
|
5951
|
+
rootSpan.setStatus({ code: SpanStatusCode.OK });
|
|
5952
|
+
} catch (error) {
|
|
5953
|
+
rootSpan.setStatus({
|
|
5954
|
+
code: SpanStatusCode.ERROR,
|
|
5955
|
+
message: error instanceof Error ? error.message : String(error)
|
|
5956
|
+
});
|
|
5957
|
+
rootSpan.recordException(error instanceof Error ? error : new Error(String(error)));
|
|
5958
|
+
throw error;
|
|
5959
|
+
} finally {
|
|
5960
|
+
this.insideTelemetryCapture = false;
|
|
5961
|
+
rootSpan.end();
|
|
5962
|
+
try {
|
|
5963
|
+
await this.otlpSpanProcessor?.forceFlush();
|
|
5964
|
+
} catch {
|
|
5965
|
+
}
|
|
5966
|
+
}
|
|
5786
5967
|
} else {
|
|
5787
5968
|
yield* this.run(input);
|
|
5788
5969
|
}
|
|
@@ -6271,7 +6452,7 @@ ${textContent}` };
|
|
|
6271
6452
|
abortSignal: input.abortSignal,
|
|
6272
6453
|
...typeof maxTokens === "number" ? { maxTokens } : {},
|
|
6273
6454
|
experimental_telemetry: {
|
|
6274
|
-
isEnabled: telemetryEnabled && !!this.latitudeTelemetry,
|
|
6455
|
+
isEnabled: telemetryEnabled && !!(this.latitudeTelemetry || this.hasOtlpExporter),
|
|
6275
6456
|
recordInputs: true,
|
|
6276
6457
|
recordOutputs: true
|
|
6277
6458
|
}
|
|
@@ -6442,6 +6623,13 @@ ${textContent}` };
|
|
|
6442
6623
|
`[poncho][harness] Model "${modelName}" returned an empty response with finishReason="stop" on step ${step}.`
|
|
6443
6624
|
);
|
|
6444
6625
|
}
|
|
6626
|
+
if (fullText.length > 0) {
|
|
6627
|
+
messages.push({
|
|
6628
|
+
role: "assistant",
|
|
6629
|
+
content: fullText,
|
|
6630
|
+
metadata: { timestamp: now(), id: randomUUID3(), step }
|
|
6631
|
+
});
|
|
6632
|
+
}
|
|
6445
6633
|
responseText = fullText;
|
|
6446
6634
|
yield pushEvent({
|
|
6447
6635
|
type: "step:completed",
|
|
@@ -6459,7 +6647,8 @@ ${textContent}` };
|
|
|
6459
6647
|
},
|
|
6460
6648
|
duration: now() - start,
|
|
6461
6649
|
contextTokens: latestContextTokens + toolOutputEstimateSinceModel,
|
|
6462
|
-
contextWindow
|
|
6650
|
+
contextWindow,
|
|
6651
|
+
continuationMessages: [...messages]
|
|
6463
6652
|
};
|
|
6464
6653
|
yield pushEvent({ type: "run:completed", runId, result: result2 });
|
|
6465
6654
|
return;
|
|
@@ -6555,6 +6744,27 @@ ${textContent}` };
|
|
|
6555
6744
|
})
|
|
6556
6745
|
);
|
|
6557
6746
|
}
|
|
6747
|
+
} else if (this.insideTelemetryCapture && this.hasOtlpExporter) {
|
|
6748
|
+
const tracer = trace.getTracer("poncho");
|
|
6749
|
+
for (const call of approvedCalls) {
|
|
6750
|
+
const span = tracer.startSpan(`tool ${call.name}`, {
|
|
6751
|
+
attributes: {
|
|
6752
|
+
"poncho.tool.name": call.name,
|
|
6753
|
+
"poncho.tool.call_id": call.id,
|
|
6754
|
+
"poncho.tool.arguments": JSON.stringify(call.input)
|
|
6755
|
+
}
|
|
6756
|
+
});
|
|
6757
|
+
toolSpans.set(call.id, {
|
|
6758
|
+
end(opts) {
|
|
6759
|
+
if (opts.result.isError) {
|
|
6760
|
+
span.setStatus({ code: SpanStatusCode.ERROR, message: String(opts.result.value) });
|
|
6761
|
+
} else {
|
|
6762
|
+
span.setStatus({ code: SpanStatusCode.OK });
|
|
6763
|
+
}
|
|
6764
|
+
span.end();
|
|
6765
|
+
}
|
|
6766
|
+
});
|
|
6767
|
+
}
|
|
6558
6768
|
}
|
|
6559
6769
|
const TOOL_DEADLINE_SENTINEL = /* @__PURE__ */ Symbol("tool_deadline");
|
|
6560
6770
|
const toolDeadlineRemainingMs = softDeadlineMs > 0 ? softDeadlineMs - (now() - start) : Infinity;
|
|
@@ -8066,73 +8276,6 @@ var createConversationStore = (config, options) => {
|
|
|
8066
8276
|
return new InMemoryConversationStore(ttl);
|
|
8067
8277
|
};
|
|
8068
8278
|
|
|
8069
|
-
// src/telemetry.ts
|
|
8070
|
-
var MAX_FIELD_LENGTH = 200;
|
|
8071
|
-
function sanitizeEventForLog(event) {
|
|
8072
|
-
return JSON.stringify(event, (_key, value) => {
|
|
8073
|
-
if (typeof value === "string" && value.length > MAX_FIELD_LENGTH) {
|
|
8074
|
-
return `${value.slice(0, 80)}...[${value.length} chars]`;
|
|
8075
|
-
}
|
|
8076
|
-
return value;
|
|
8077
|
-
});
|
|
8078
|
-
}
|
|
8079
|
-
var TelemetryEmitter = class {
|
|
8080
|
-
config;
|
|
8081
|
-
constructor(config) {
|
|
8082
|
-
this.config = config;
|
|
8083
|
-
}
|
|
8084
|
-
async emit(event) {
|
|
8085
|
-
if (this.config?.enabled === false) {
|
|
8086
|
-
return;
|
|
8087
|
-
}
|
|
8088
|
-
if (this.config?.handler) {
|
|
8089
|
-
await this.config.handler(event);
|
|
8090
|
-
return;
|
|
8091
|
-
}
|
|
8092
|
-
if (this.config?.otlp) {
|
|
8093
|
-
await this.sendOtlp(event);
|
|
8094
|
-
}
|
|
8095
|
-
process.stdout.write(`[event] ${event.type} ${sanitizeEventForLog(event)}
|
|
8096
|
-
`);
|
|
8097
|
-
}
|
|
8098
|
-
async sendOtlp(event) {
|
|
8099
|
-
const endpoint = this.config?.otlp;
|
|
8100
|
-
if (!endpoint) {
|
|
8101
|
-
return;
|
|
8102
|
-
}
|
|
8103
|
-
try {
|
|
8104
|
-
await fetch(endpoint, {
|
|
8105
|
-
method: "POST",
|
|
8106
|
-
headers: { "Content-Type": "application/json" },
|
|
8107
|
-
body: JSON.stringify({
|
|
8108
|
-
resourceLogs: [
|
|
8109
|
-
{
|
|
8110
|
-
scopeLogs: [
|
|
8111
|
-
{
|
|
8112
|
-
logRecords: [
|
|
8113
|
-
{
|
|
8114
|
-
timeUnixNano: String(Date.now() * 1e6),
|
|
8115
|
-
severityText: "INFO",
|
|
8116
|
-
body: { stringValue: event.type },
|
|
8117
|
-
attributes: [
|
|
8118
|
-
{
|
|
8119
|
-
key: "event.payload",
|
|
8120
|
-
value: { stringValue: JSON.stringify(event) }
|
|
8121
|
-
}
|
|
8122
|
-
]
|
|
8123
|
-
}
|
|
8124
|
-
]
|
|
8125
|
-
}
|
|
8126
|
-
]
|
|
8127
|
-
}
|
|
8128
|
-
]
|
|
8129
|
-
})
|
|
8130
|
-
});
|
|
8131
|
-
} catch {
|
|
8132
|
-
}
|
|
8133
|
-
}
|
|
8134
|
-
};
|
|
8135
|
-
|
|
8136
8279
|
// src/index.ts
|
|
8137
8280
|
import { defineTool as defineTool7 } from "@poncho-ai/sdk";
|
|
8138
8281
|
export {
|
|
@@ -8180,6 +8323,7 @@ export {
|
|
|
8180
8323
|
loadSkillContext,
|
|
8181
8324
|
loadSkillInstructions,
|
|
8182
8325
|
loadSkillMetadata,
|
|
8326
|
+
normalizeOtlp,
|
|
8183
8327
|
normalizeScriptPolicyPath,
|
|
8184
8328
|
parseAgentFile,
|
|
8185
8329
|
parseAgentMarkdown,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@poncho-ai/harness",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.29.0",
|
|
4
4
|
"description": "Agent execution runtime - conversation loop, tool dispatch, streaming",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -25,6 +25,8 @@
|
|
|
25
25
|
"@aws-sdk/client-dynamodb": "^3.988.0",
|
|
26
26
|
"@latitude-data/telemetry": "^2.0.4",
|
|
27
27
|
"@opentelemetry/api": "1.9.0",
|
|
28
|
+
"@opentelemetry/exporter-trace-otlp-http": "^0.213.0",
|
|
29
|
+
"@opentelemetry/sdk-trace-node": "^2.6.0",
|
|
28
30
|
"ai": "^6.0.86",
|
|
29
31
|
"cheerio": "^1.2.0",
|
|
30
32
|
"jiti": "^2.6.1",
|
|
@@ -32,7 +34,7 @@
|
|
|
32
34
|
"redis": "^5.10.0",
|
|
33
35
|
"yaml": "^2.4.0",
|
|
34
36
|
"zod": "^3.22.0",
|
|
35
|
-
"@poncho-ai/sdk": "1.6.
|
|
37
|
+
"@poncho-ai/sdk": "1.6.2"
|
|
36
38
|
},
|
|
37
39
|
"devDependencies": {
|
|
38
40
|
"@types/mustache": "^4.2.6",
|
package/src/config.ts
CHANGED
|
@@ -104,7 +104,10 @@ export interface PonchoConfig extends McpConfig {
|
|
|
104
104
|
};
|
|
105
105
|
telemetry?: {
|
|
106
106
|
enabled?: boolean;
|
|
107
|
-
otlp?: string
|
|
107
|
+
otlp?: string | {
|
|
108
|
+
url: string;
|
|
109
|
+
headers?: Record<string, string>;
|
|
110
|
+
};
|
|
108
111
|
latitude?: {
|
|
109
112
|
apiKeyEnv?: string;
|
|
110
113
|
projectIdEnv?: string;
|
package/src/harness.ts
CHANGED
|
@@ -36,6 +36,10 @@ import { createSearchTools } from "./search-tools.js";
|
|
|
36
36
|
import { createSubagentTools } from "./subagent-tools.js";
|
|
37
37
|
import type { SubagentManager } from "./subagent-manager.js";
|
|
38
38
|
import { LatitudeTelemetry } from "@latitude-data/telemetry";
|
|
39
|
+
import { trace, context as otelContext, SpanStatusCode } from "@opentelemetry/api";
|
|
40
|
+
import { NodeTracerProvider, BatchSpanProcessor } from "@opentelemetry/sdk-trace-node";
|
|
41
|
+
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
|
|
42
|
+
import { normalizeOtlp } from "./telemetry.js";
|
|
39
43
|
import {
|
|
40
44
|
isSiblingScriptsPattern,
|
|
41
45
|
matchesRelativeScriptPattern,
|
|
@@ -560,6 +564,9 @@ export class AgentHarness {
|
|
|
560
564
|
private readonly activeSkillNames = new Set<string>();
|
|
561
565
|
private readonly registeredMcpToolNames = new Set<string>();
|
|
562
566
|
private latitudeTelemetry?: LatitudeTelemetry;
|
|
567
|
+
private otlpSpanProcessor?: BatchSpanProcessor;
|
|
568
|
+
private otlpTracerProvider?: NodeTracerProvider;
|
|
569
|
+
private hasOtlpExporter = false;
|
|
563
570
|
private insideTelemetryCapture = false;
|
|
564
571
|
private _browserSession?: unknown;
|
|
565
572
|
private _browserMod?: {
|
|
@@ -1079,6 +1086,37 @@ export class AgentHarness {
|
|
|
1079
1086
|
`[poncho][telemetry] Latitude telemetry is configured but missing: ${missing.join(", ")}. Traces will NOT be sent.`,
|
|
1080
1087
|
);
|
|
1081
1088
|
}
|
|
1089
|
+
|
|
1090
|
+
// Generic OTLP trace exporter — works alongside or instead of Latitude.
|
|
1091
|
+
const otlpConfig = telemetryEnabled ? normalizeOtlp(config?.telemetry?.otlp) : undefined;
|
|
1092
|
+
if (otlpConfig) {
|
|
1093
|
+
const exporter = new OTLPTraceExporter({
|
|
1094
|
+
url: otlpConfig.url,
|
|
1095
|
+
headers: otlpConfig.headers,
|
|
1096
|
+
});
|
|
1097
|
+
const processor = new BatchSpanProcessor(exporter);
|
|
1098
|
+
this.otlpSpanProcessor = processor;
|
|
1099
|
+
|
|
1100
|
+
if (this.latitudeTelemetry) {
|
|
1101
|
+
// Latitude already registered a global TracerProvider (v1.x) — add our
|
|
1102
|
+
// processor to it so every span flows to both destinations.
|
|
1103
|
+
const globalProvider = trace.getTracerProvider();
|
|
1104
|
+
const delegate = (globalProvider as unknown as { getDelegate?: () => unknown })
|
|
1105
|
+
.getDelegate?.() ?? globalProvider;
|
|
1106
|
+
if (typeof (delegate as Record<string, unknown>).addSpanProcessor === "function") {
|
|
1107
|
+
(delegate as unknown as { addSpanProcessor(p: BatchSpanProcessor): void }).addSpanProcessor(processor);
|
|
1108
|
+
}
|
|
1109
|
+
console.info(`[poncho][telemetry] OTLP exporter added (piggybacking on Latitude provider) → ${otlpConfig.url}`);
|
|
1110
|
+
} else {
|
|
1111
|
+
const provider = new NodeTracerProvider({
|
|
1112
|
+
spanProcessors: [processor],
|
|
1113
|
+
});
|
|
1114
|
+
provider.register();
|
|
1115
|
+
this.otlpTracerProvider = provider;
|
|
1116
|
+
console.info(`[poncho][telemetry] OTLP exporter active (standalone provider) → ${otlpConfig.url}`);
|
|
1117
|
+
}
|
|
1118
|
+
this.hasOtlpExporter = true;
|
|
1119
|
+
}
|
|
1082
1120
|
}
|
|
1083
1121
|
|
|
1084
1122
|
private async buildBrowserStoragePersistence(
|
|
@@ -1250,6 +1288,27 @@ export class AgentHarness {
|
|
|
1250
1288
|
});
|
|
1251
1289
|
this.latitudeTelemetry = undefined;
|
|
1252
1290
|
}
|
|
1291
|
+
if (this.otlpSpanProcessor) {
|
|
1292
|
+
await this.otlpSpanProcessor.shutdown().catch((err) => {
|
|
1293
|
+
console.warn(
|
|
1294
|
+
`[poncho][telemetry] OTLP span processor shutdown error: ${
|
|
1295
|
+
err instanceof Error ? err.message : String(err)
|
|
1296
|
+
}`,
|
|
1297
|
+
);
|
|
1298
|
+
});
|
|
1299
|
+
this.otlpSpanProcessor = undefined;
|
|
1300
|
+
}
|
|
1301
|
+
if (this.otlpTracerProvider) {
|
|
1302
|
+
await this.otlpTracerProvider.shutdown().catch((err) => {
|
|
1303
|
+
console.warn(
|
|
1304
|
+
`[poncho][telemetry] OTLP tracer provider shutdown error: ${
|
|
1305
|
+
err instanceof Error ? err.message : String(err)
|
|
1306
|
+
}`,
|
|
1307
|
+
);
|
|
1308
|
+
});
|
|
1309
|
+
this.otlpTracerProvider = undefined;
|
|
1310
|
+
}
|
|
1311
|
+
this.hasOtlpExporter = false;
|
|
1253
1312
|
}
|
|
1254
1313
|
|
|
1255
1314
|
listTools(): ToolDefinition[] {
|
|
@@ -1257,18 +1316,20 @@ export class AgentHarness {
|
|
|
1257
1316
|
}
|
|
1258
1317
|
|
|
1259
1318
|
/**
|
|
1260
|
-
* Wraps the run() generator with
|
|
1261
|
-
*
|
|
1319
|
+
* Wraps the run() generator with telemetry capture for complete trace coverage.
|
|
1320
|
+
* Supports Latitude, generic OTLP, or both simultaneously.
|
|
1321
|
+
* Streams events in real-time using an event queue pattern.
|
|
1262
1322
|
*/
|
|
1263
1323
|
async *runWithTelemetry(input: RunInput): AsyncGenerator<AgentEvent> {
|
|
1264
1324
|
const config = this.loadedConfig;
|
|
1265
1325
|
const telemetry = this.latitudeTelemetry;
|
|
1266
1326
|
|
|
1267
1327
|
if (telemetry) {
|
|
1328
|
+
// Latitude capture path — wraps run() inside telemetry.capture().
|
|
1329
|
+
// If OTLP is also configured, spans flow to both via the shared provider.
|
|
1268
1330
|
const latProjectIdEnv2 = config?.telemetry?.latitude?.projectIdEnv ?? "LATITUDE_PROJECT_ID";
|
|
1269
1331
|
const projectId = parseInt(process.env[latProjectIdEnv2] ?? "", 10) as number;
|
|
1270
1332
|
const rawPath = config?.telemetry?.latitude?.path ?? this.parsedAgent?.frontmatter.name ?? 'agent';
|
|
1271
|
-
// Sanitize path for Latitude's DOCUMENT_PATH_REGEXP: /^([\w-]+\/)*([\w-.])+$/
|
|
1272
1333
|
const path = rawPath.replace(/[^\w\-./]/g, '-').replace(/-+/g, '-').replace(/^-+|-+$/g, '') || 'agent';
|
|
1273
1334
|
|
|
1274
1335
|
const rawConversationId = input.conversationId ?? (
|
|
@@ -1276,7 +1337,6 @@ export class AgentHarness {
|
|
|
1276
1337
|
? input.parameters.__activeConversationId
|
|
1277
1338
|
: undefined
|
|
1278
1339
|
);
|
|
1279
|
-
// Latitude expects a UUID v4 for documentLogUuid; only pass it if valid
|
|
1280
1340
|
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
1281
1341
|
const conversationUuid = rawConversationId && UUID_RE.test(rawConversationId)
|
|
1282
1342
|
? rawConversationId
|
|
@@ -1286,13 +1346,11 @@ export class AgentHarness {
|
|
|
1286
1346
|
`[poncho][telemetry] Latitude telemetry active – projectId=${projectId}, path="${path}"${conversationUuid ? `, conversation="${conversationUuid}"` : ""}`,
|
|
1287
1347
|
);
|
|
1288
1348
|
|
|
1289
|
-
// Event queue for streaming events in real-time
|
|
1290
1349
|
const eventQueue: AgentEvent[] = [];
|
|
1291
1350
|
let queueResolve: ((value: void) => void) | null = null;
|
|
1292
1351
|
let generatorDone = false;
|
|
1293
1352
|
let generatorError: Error | null = null;
|
|
1294
1353
|
|
|
1295
|
-
// Start the generator inside telemetry.capture() (runs in background)
|
|
1296
1354
|
const capturePromise = telemetry.capture({ projectId, path, conversationUuid }, async () => {
|
|
1297
1355
|
this.insideTelemetryCapture = true;
|
|
1298
1356
|
try {
|
|
@@ -1316,13 +1374,11 @@ export class AgentHarness {
|
|
|
1316
1374
|
}
|
|
1317
1375
|
});
|
|
1318
1376
|
|
|
1319
|
-
// Yield events from the queue as they arrive
|
|
1320
1377
|
try {
|
|
1321
1378
|
while (!generatorDone || eventQueue.length > 0) {
|
|
1322
1379
|
if (eventQueue.length > 0) {
|
|
1323
1380
|
yield eventQueue.shift()!;
|
|
1324
1381
|
} else if (!generatorDone) {
|
|
1325
|
-
// Wait for next event
|
|
1326
1382
|
await new Promise<void>((resolve) => {
|
|
1327
1383
|
queueResolve = resolve;
|
|
1328
1384
|
});
|
|
@@ -1344,8 +1400,47 @@ export class AgentHarness {
|
|
|
1344
1400
|
}
|
|
1345
1401
|
}
|
|
1346
1402
|
}
|
|
1403
|
+
} else if (this.hasOtlpExporter) {
|
|
1404
|
+
// Standalone OTLP path — create a root span for the agent run so all
|
|
1405
|
+
// child spans (LLM calls via Vercel AI SDK, tool spans) are grouped
|
|
1406
|
+
// under a single trace.
|
|
1407
|
+
const tracer = trace.getTracer("poncho");
|
|
1408
|
+
const agentName = this.parsedAgent?.frontmatter.name ?? "agent";
|
|
1409
|
+
|
|
1410
|
+
const rootSpan = tracer.startSpan(`agent.run ${agentName}`);
|
|
1411
|
+
rootSpan.setAttribute("poncho.agent.name", agentName);
|
|
1412
|
+
if (input.conversationId) {
|
|
1413
|
+
rootSpan.setAttribute("poncho.conversation.id", input.conversationId);
|
|
1414
|
+
}
|
|
1415
|
+
|
|
1416
|
+
// Bind the root span's context so every async step (including
|
|
1417
|
+
// streamText and tool calls) sees it as the parent span.
|
|
1418
|
+
const spanContext = trace.setSpan(otelContext.active(), rootSpan);
|
|
1419
|
+
this.insideTelemetryCapture = true;
|
|
1420
|
+
|
|
1421
|
+
try {
|
|
1422
|
+
const gen = this.run(input);
|
|
1423
|
+
let next: IteratorResult<AgentEvent>;
|
|
1424
|
+
do {
|
|
1425
|
+
next = await otelContext.with(spanContext, () => gen.next());
|
|
1426
|
+
if (!next.done) yield next.value;
|
|
1427
|
+
} while (!next.done);
|
|
1428
|
+
rootSpan.setStatus({ code: SpanStatusCode.OK });
|
|
1429
|
+
} catch (error) {
|
|
1430
|
+
rootSpan.setStatus({
|
|
1431
|
+
code: SpanStatusCode.ERROR,
|
|
1432
|
+
message: error instanceof Error ? error.message : String(error),
|
|
1433
|
+
});
|
|
1434
|
+
rootSpan.recordException(error instanceof Error ? error : new Error(String(error)));
|
|
1435
|
+
throw error;
|
|
1436
|
+
} finally {
|
|
1437
|
+
this.insideTelemetryCapture = false;
|
|
1438
|
+
rootSpan.end();
|
|
1439
|
+
try {
|
|
1440
|
+
await this.otlpSpanProcessor?.forceFlush();
|
|
1441
|
+
} catch { /* best-effort */ }
|
|
1442
|
+
}
|
|
1347
1443
|
} else {
|
|
1348
|
-
// No telemetry configured, just pass through
|
|
1349
1444
|
yield* this.run(input);
|
|
1350
1445
|
}
|
|
1351
1446
|
}
|
|
@@ -1941,7 +2036,7 @@ ${boundedMainMemory.trim()}`
|
|
|
1941
2036
|
abortSignal: input.abortSignal,
|
|
1942
2037
|
...(typeof maxTokens === "number" ? { maxTokens } : {}),
|
|
1943
2038
|
experimental_telemetry: {
|
|
1944
|
-
isEnabled: telemetryEnabled && !!this.latitudeTelemetry,
|
|
2039
|
+
isEnabled: telemetryEnabled && !!(this.latitudeTelemetry || this.hasOtlpExporter),
|
|
1945
2040
|
recordInputs: true,
|
|
1946
2041
|
recordOutputs: true,
|
|
1947
2042
|
},
|
|
@@ -2138,6 +2233,13 @@ ${boundedMainMemory.trim()}`
|
|
|
2138
2233
|
`[poncho][harness] Model "${modelName}" returned an empty response with finishReason="stop" on step ${step}.`,
|
|
2139
2234
|
);
|
|
2140
2235
|
}
|
|
2236
|
+
if (fullText.length > 0) {
|
|
2237
|
+
messages.push({
|
|
2238
|
+
role: "assistant",
|
|
2239
|
+
content: fullText,
|
|
2240
|
+
metadata: { timestamp: now(), id: randomUUID(), step },
|
|
2241
|
+
});
|
|
2242
|
+
}
|
|
2141
2243
|
responseText = fullText;
|
|
2142
2244
|
yield pushEvent({
|
|
2143
2245
|
type: "step:completed",
|
|
@@ -2156,6 +2258,7 @@ ${boundedMainMemory.trim()}`
|
|
|
2156
2258
|
duration: now() - start,
|
|
2157
2259
|
contextTokens: latestContextTokens + toolOutputEstimateSinceModel,
|
|
2158
2260
|
contextWindow,
|
|
2261
|
+
continuationMessages: [...messages],
|
|
2159
2262
|
};
|
|
2160
2263
|
yield pushEvent({ type: "run:completed", runId, result });
|
|
2161
2264
|
return;
|
|
@@ -2275,7 +2378,7 @@ ${boundedMainMemory.trim()}`
|
|
|
2275
2378
|
return;
|
|
2276
2379
|
}
|
|
2277
2380
|
|
|
2278
|
-
// Create telemetry tool spans so tool calls appear in
|
|
2381
|
+
// Create telemetry tool spans so tool calls appear in traces
|
|
2279
2382
|
type ToolSpanHandle = { end: (opts: { result: { value: unknown; isError: boolean } }) => void };
|
|
2280
2383
|
const toolSpans = new Map<string, ToolSpanHandle>();
|
|
2281
2384
|
if (this.insideTelemetryCapture && this.latitudeTelemetry) {
|
|
@@ -2288,6 +2391,27 @@ ${boundedMainMemory.trim()}`
|
|
|
2288
2391
|
}),
|
|
2289
2392
|
);
|
|
2290
2393
|
}
|
|
2394
|
+
} else if (this.insideTelemetryCapture && this.hasOtlpExporter) {
|
|
2395
|
+
const tracer = trace.getTracer("poncho");
|
|
2396
|
+
for (const call of approvedCalls) {
|
|
2397
|
+
const span = tracer.startSpan(`tool ${call.name}`, {
|
|
2398
|
+
attributes: {
|
|
2399
|
+
"poncho.tool.name": call.name,
|
|
2400
|
+
"poncho.tool.call_id": call.id,
|
|
2401
|
+
"poncho.tool.arguments": JSON.stringify(call.input),
|
|
2402
|
+
},
|
|
2403
|
+
});
|
|
2404
|
+
toolSpans.set(call.id, {
|
|
2405
|
+
end(opts: { result: { value: unknown; isError: boolean } }) {
|
|
2406
|
+
if (opts.result.isError) {
|
|
2407
|
+
span.setStatus({ code: SpanStatusCode.ERROR, message: String(opts.result.value) });
|
|
2408
|
+
} else {
|
|
2409
|
+
span.setStatus({ code: SpanStatusCode.OK });
|
|
2410
|
+
}
|
|
2411
|
+
span.end();
|
|
2412
|
+
},
|
|
2413
|
+
});
|
|
2414
|
+
}
|
|
2291
2415
|
}
|
|
2292
2416
|
|
|
2293
2417
|
// Race tool execution against the soft deadline so long-running tool
|
package/src/state.ts
CHANGED
|
@@ -71,6 +71,11 @@ export interface Conversation {
|
|
|
71
71
|
/** Harness-internal message chain preserved across continuation runs.
|
|
72
72
|
* Cleared when a run completes without continuation. */
|
|
73
73
|
_continuationMessages?: Message[];
|
|
74
|
+
/** Full structured message chain from the last harness run, including
|
|
75
|
+
* tool-call and tool-result messages the model needs for context.
|
|
76
|
+
* Unlike `_continuationMessages`, this is always set after a run
|
|
77
|
+
* and does NOT signal that a continuation is pending. */
|
|
78
|
+
_harnessMessages?: Message[];
|
|
74
79
|
createdAt: number;
|
|
75
80
|
updatedAt: number;
|
|
76
81
|
}
|
package/src/telemetry.ts
CHANGED
|
@@ -11,9 +11,22 @@ function sanitizeEventForLog(event: AgentEvent): string {
|
|
|
11
11
|
});
|
|
12
12
|
}
|
|
13
13
|
|
|
14
|
+
export interface OtlpConfig {
|
|
15
|
+
url: string;
|
|
16
|
+
headers?: Record<string, string>;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export type OtlpOption = string | OtlpConfig;
|
|
20
|
+
|
|
21
|
+
export function normalizeOtlp(opt: OtlpOption | undefined): OtlpConfig | undefined {
|
|
22
|
+
if (!opt) return undefined;
|
|
23
|
+
if (typeof opt === "string") return opt ? { url: opt } : undefined;
|
|
24
|
+
return opt.url ? opt : undefined;
|
|
25
|
+
}
|
|
26
|
+
|
|
14
27
|
export interface TelemetryConfig {
|
|
15
28
|
enabled?: boolean;
|
|
16
|
-
otlp?:
|
|
29
|
+
otlp?: OtlpOption;
|
|
17
30
|
latitude?: {
|
|
18
31
|
apiKeyEnv?: string;
|
|
19
32
|
projectIdEnv?: string;
|
|
@@ -38,8 +51,9 @@ export class TelemetryEmitter {
|
|
|
38
51
|
await this.config.handler(event);
|
|
39
52
|
return;
|
|
40
53
|
}
|
|
41
|
-
|
|
42
|
-
|
|
54
|
+
const otlp = normalizeOtlp(this.config?.otlp);
|
|
55
|
+
if (otlp) {
|
|
56
|
+
await this.sendOtlp(event, otlp);
|
|
43
57
|
}
|
|
44
58
|
// Latitude telemetry is handled by LatitudeTelemetry (from
|
|
45
59
|
// @latitude-data/telemetry) via harness.runWithTelemetry().
|
|
@@ -48,15 +62,11 @@ export class TelemetryEmitter {
|
|
|
48
62
|
process.stdout.write(`[event] ${event.type} ${sanitizeEventForLog(event)}\n`);
|
|
49
63
|
}
|
|
50
64
|
|
|
51
|
-
private async sendOtlp(event: AgentEvent): Promise<void> {
|
|
52
|
-
const endpoint = this.config?.otlp;
|
|
53
|
-
if (!endpoint) {
|
|
54
|
-
return;
|
|
55
|
-
}
|
|
65
|
+
private async sendOtlp(event: AgentEvent, otlp: OtlpConfig): Promise<void> {
|
|
56
66
|
try {
|
|
57
|
-
await fetch(
|
|
67
|
+
await fetch(otlp.url, {
|
|
58
68
|
method: "POST",
|
|
59
|
-
headers: { "Content-Type": "application/json" },
|
|
69
|
+
headers: { "Content-Type": "application/json", ...otlp.headers },
|
|
60
70
|
body: JSON.stringify({
|
|
61
71
|
resourceLogs: [
|
|
62
72
|
{
|