@companion-ai/feynman 0.5.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/README.md +7 -1
- package/RELEASES.md +8 -0
- package/dist/cli.js +4 -1
- package/dist/pi/launch.js +5 -0
- package/dist/pi/runtime.js +2 -0
- package/dist/setup/doctor.js +2 -0
- package/dist/telemetry/posthog.js +44 -1
- package/extensions/research-tools/telemetry.ts +263 -0
- package/extensions/research-tools.ts +2 -0
- package/package.json +1 -1
package/AGENTS.md
CHANGED
|
@@ -84,7 +84,7 @@ Never use generic names like `research.md`, `draft.md`, `brief.md`, or `summary.
|
|
|
84
84
|
|
|
85
85
|
## Workspace changelog
|
|
86
86
|
|
|
87
|
-
- `CHANGELOG.md` is a lab notebook, not release notes.
|
|
87
|
+
- `CHANGELOG.md` is a local, untracked lab notebook (ignored by git), not release notes. Release notes live in `RELEASES.md`.
|
|
88
88
|
- Read `CHANGELOG.md` before resuming substantial work when it exists.
|
|
89
89
|
- Append concise entries after meaningful progress, failed approaches, major verification results, or new blockers.
|
|
90
90
|
- Each entry should identify the active slug or objective and end with the next recommended step.
|
package/README.md
CHANGED
|
@@ -188,7 +188,7 @@ Four bundled research agents, invoked by workflow prompts when decomposition hel
|
|
|
188
188
|
- **[Hugging Face Hub](https://huggingface.co/docs/hub/api)** — dataset metadata, split/schema inspection, and small file reads from model, dataset, and Space repos
|
|
189
189
|
- **Web research** — multi-provider search, explicit proxy routing, bounded GitHub issue/PR documents, raw or question-grounded page retrieval, direct images, external fetched-content caching, stored-page passage lookup, and auditable source text; tools, commands, images, PDFs, and browser cookies remain independently gated
|
|
190
190
|
- **Session search** — indexed recall across prior research sessions
|
|
191
|
-
- **Observability** — opt-out PostHog
|
|
191
|
+
- **Observability** — opt-out PostHog usage metadata for CLI commands, research workflows, tools, and model calls (see [Telemetry](#telemetry))
|
|
192
192
|
- **Research execution options** — Docker, plus Modal or RunPod when their CLIs are installed, for explicitly chosen replication, benchmark, or dataset-heavy experiment runs; not service deployment or generic cloud administration
|
|
193
193
|
|
|
194
194
|
---
|
|
@@ -199,6 +199,12 @@ Built on [Pi](https://github.com/badlogic/pi-mono) for the agent runtime, [alpha
|
|
|
199
199
|
|
|
200
200
|
---
|
|
201
201
|
|
|
202
|
+
### Telemetry
|
|
203
|
+
|
|
204
|
+
Feynman sends anonymous usage telemetry to PostHog by default and prints a one-time notice on first run. It sends commands, workflow names and outcomes, tool names, model and provider names, token counts, latency, and error flags under a random install ID. It never sends prompts, model output, paper content, file paths, or tool arguments. Set `FEYNMAN_TELEMETRY=off` (or `DO_NOT_TRACK=1`) to disable it; `feynman status` shows the current setting. The full event list is in the [configuration docs](https://feynman.is/docs/getting-started/configuration#telemetry).
|
|
205
|
+
|
|
206
|
+
---
|
|
207
|
+
|
|
202
208
|
### Star History
|
|
203
209
|
|
|
204
210
|
<a href="https://www.star-history.com/?repos=Companion-Inc%2Ffeynman&type=date&legend=top-left">
|
package/RELEASES.md
CHANGED
|
@@ -6,6 +6,14 @@ GitHub release notes are generated from the matching `## vX.Y.Z` section in this
|
|
|
6
6
|
|
|
7
7
|
## Unreleased
|
|
8
8
|
|
|
9
|
+
## v0.5.1 - 2026-09-23
|
|
10
|
+
|
|
11
|
+
### Telemetry
|
|
12
|
+
|
|
13
|
+
- **Research runs are now visible in telemetry, without content.** The research extension inside Pi sends `feynman_session_started`, `feynman_workflow_started` and `feynman_workflow_completed` (workflow name such as `deepresearch` or `chat`, status, tool and subagent call counts, whether anything under `outputs/` or `papers/` was written, duration), `feynman_tool_used` (tool name and error flag), and one PostHog LLM analytics `$ai_generation` event per model response (model, provider, token counts, latency, HTTP status, stop reason; the trace ID is the Pi session ID). Prompts, model output, paper content, file paths, and tool arguments are never sent.
|
|
14
|
+
- **First-run notice.** The first interactive run prints a one-time notice on stderr (never in scripts or CI) saying telemetry is on, what it collects, and how to turn it off with `FEYNMAN_TELEMETRY=off`. `feynman status` shows whether telemetry is on.
|
|
15
|
+
- Telemetry stays on by default and follows the existing opt-outs: `FEYNMAN_TELEMETRY=off` or `DO_NOT_TRACK=1` disables the CLI and the Pi runtime events alike.
|
|
16
|
+
|
|
9
17
|
## v0.5.0 - 2026-09-23
|
|
10
18
|
|
|
11
19
|
### Stock Pi runtime
|
package/dist/cli.js
CHANGED
|
@@ -29,7 +29,7 @@ import { fetchLatestFeynmanVersion, getFeynmanUpgradeLines, isNewerVersion } fro
|
|
|
29
29
|
import { runDoctor, runStatus } from "./setup/doctor.js";
|
|
30
30
|
import { setupPreviewDependencies } from "./setup/preview.js";
|
|
31
31
|
import { runSetup } from "./setup/setup.js";
|
|
32
|
-
import { captureTelemetryEvent, emitTelemetryLog, getCliTelemetryMetadata, initializePostHogTelemetry, shutdownPostHogTelemetry, startTelemetrySpan, telemetryErrorProperties, } from "./telemetry/posthog.js";
|
|
32
|
+
import { captureTelemetryEvent, emitTelemetryLog, getCliTelemetryMetadata, initializePostHogTelemetry, shutdownPostHogTelemetry, startTelemetrySpan, telemetryErrorProperties, telemetryFirstRunNotice, } from "./telemetry/posthog.js";
|
|
33
33
|
import { ASH, printAsciiHeader, printInfo, printPanel, printSection, RESET, SAGE } from "./ui/terminal.js";
|
|
34
34
|
import { createModelRuntime } from "./model/registry.js";
|
|
35
35
|
import { cliCommandSections, formatCliWorkflowUsage, legacyFlags, readPromptSpecs, topLevelCommandNames, } from "../metadata/commands.mjs";
|
|
@@ -386,6 +386,9 @@ export async function main() {
|
|
|
386
386
|
const appRoot = resolve(here, "..");
|
|
387
387
|
const feynmanVersion = loadPackageVersion(appRoot).version;
|
|
388
388
|
initializePostHogTelemetry({ appVersion: feynmanVersion, serviceName: "feynman-cli" });
|
|
389
|
+
const telemetryNotice = telemetryFirstRunNotice();
|
|
390
|
+
if (telemetryNotice)
|
|
391
|
+
process.stderr.write(`${telemetryNotice}\n`);
|
|
389
392
|
const commandTelemetry = getCliTelemetryMetadata(process.argv.slice(2), { knownCommands: getTelemetryCommandNames(appRoot) });
|
|
390
393
|
const commandStartedAt = Date.now();
|
|
391
394
|
const commandSpan = startTelemetrySpan("feynman.cli.command", commandTelemetry);
|
package/dist/pi/launch.js
CHANGED
|
@@ -2,6 +2,7 @@ import { spawn } from "node:child_process";
|
|
|
2
2
|
import { constants } from "node:os";
|
|
3
3
|
import { buildPiArgs, buildPiEnv, ensureFeynmanCommandShim, ensureFeynmanWorkspaceScaffold, resolvePiCliPath, } from "./runtime.js";
|
|
4
4
|
import { resolveAllExecutables } from "../system/executables.js";
|
|
5
|
+
import { telemetryFirstRunNotice } from "../telemetry/posthog.js";
|
|
5
6
|
export function exitCodeFromSignal(signal) {
|
|
6
7
|
const signalNumber = constants.signals[signal];
|
|
7
8
|
return typeof signalNumber === "number" ? 128 + signalNumber : 1;
|
|
@@ -31,6 +32,10 @@ export async function runPi(options, args, env, stdin = "inherit") {
|
|
|
31
32
|
export async function launchPiChat(options) {
|
|
32
33
|
if (process.stdout.isTTY && options.mode !== "rpc") {
|
|
33
34
|
process.stdout.write("\x1b[2J\x1b[3J\x1b[H");
|
|
35
|
+
// Clearing the screen erased the first-run notice printed at startup.
|
|
36
|
+
const telemetryNotice = telemetryFirstRunNotice();
|
|
37
|
+
if (telemetryNotice)
|
|
38
|
+
process.stderr.write(`${telemetryNotice}\n`);
|
|
34
39
|
}
|
|
35
40
|
if (options.preLaunchNotice) {
|
|
36
41
|
process.stdout.write(`${options.preLaunchNotice}\n`);
|
package/dist/pi/runtime.js
CHANGED
|
@@ -2,6 +2,7 @@ import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "n
|
|
|
2
2
|
import { createRequire } from "node:module";
|
|
3
3
|
import { delimiter, dirname, resolve } from "node:path";
|
|
4
4
|
import { BROWSER_FALLBACK_PATHS, MERMAID_FALLBACK_PATHS, PANDOC_FALLBACK_PATHS, resolveExecutable, } from "../system/executables.js";
|
|
5
|
+
import { getPostHogChildEnv } from "../telemetry/posthog.js";
|
|
5
6
|
// Pi packages shipped as Feynman dependencies and loaded from their install
|
|
6
7
|
// paths through settings.json `packages` (Pi's documented local-path source).
|
|
7
8
|
export const BUNDLED_PI_PACKAGES = ["pi-subagents", "pi-web-access", "pi-docparser", "pi-btw"];
|
|
@@ -132,5 +133,6 @@ export function buildPiEnv(options, executables) {
|
|
|
132
133
|
PI_SKIP_VERSION_CHECK: process.env.PI_SKIP_VERSION_CHECK ?? "1",
|
|
133
134
|
MERMAID_CLI_PATH: mermaidPath,
|
|
134
135
|
PUPPETEER_EXECUTABLE_PATH: browserPath,
|
|
136
|
+
...getPostHogChildEnv(),
|
|
135
137
|
};
|
|
136
138
|
}
|
package/dist/setup/doctor.js
CHANGED
|
@@ -6,6 +6,7 @@ import { formatPiWebAccessDoctorLines, getPiWebAccessStatus } from "../pi/web-ac
|
|
|
6
6
|
import { BROWSER_FALLBACK_PATHS, PANDOC_FALLBACK_PATHS, resolveExecutable } from "../system/executables.js";
|
|
7
7
|
import { readJson } from "../pi/settings.js";
|
|
8
8
|
import { validatePiInstallation } from "../pi/runtime.js";
|
|
9
|
+
import { isTelemetryDisabled } from "../telemetry/posthog.js";
|
|
9
10
|
import { printInfo, printPanel, printSection } from "../ui/terminal.js";
|
|
10
11
|
import { getCurrentModelSpec } from "../model/commands.js";
|
|
11
12
|
import { buildModelStatusSnapshotFromRecords, getAvailableModelRecords, getSupportedModelRecords } from "../model/catalog.js";
|
|
@@ -79,6 +80,7 @@ export async function runStatus(options) {
|
|
|
79
80
|
printInfo(`Web access: pi-web-access (${snapshot.webRouteLabel})`);
|
|
80
81
|
printInfo(`Service tier: ${getConfiguredServiceTier(options.settingsPath) ?? "not set"}`);
|
|
81
82
|
printInfo(`Preview: ${snapshot.previewConfigured ? "configured" : "not configured"}`);
|
|
83
|
+
printInfo(`Telemetry: ${isTelemetryDisabled() ? "off" : "on (anonymous usage metadata; set FEYNMAN_TELEMETRY=off to disable)"}`);
|
|
82
84
|
printSection("Paths");
|
|
83
85
|
printInfo(`Sessions: ${snapshot.sessionDir}`);
|
|
84
86
|
printSection("Runtime");
|
|
@@ -19,6 +19,11 @@ export const DEFAULT_POSTHOG_PROJECT_TOKEN = "phc_owCZbr7c4mchCuVN5JXA6uBByjbT2k
|
|
|
19
19
|
const TELEMETRY_STATE_FILE = "telemetry.json";
|
|
20
20
|
const TELEMETRY_DISABLED_VALUES = new Set(["0", "false", "no", "off", "disabled"]);
|
|
21
21
|
const TELEMETRY_KEY_PATTERN = /^[A-Za-z0-9_$./-]+$/;
|
|
22
|
+
export const TELEMETRY_NOTICE = [
|
|
23
|
+
"Attention: Feynman collects anonymous usage telemetry: commands, workflows, tool names, models, token counts, and errors.",
|
|
24
|
+
"It never sends prompts, paper content, file paths, or tool arguments.",
|
|
25
|
+
"To opt out, set FEYNMAN_TELEMETRY=off. Learn more: https://www.feynman.is/docs/getting-started/configuration#telemetry",
|
|
26
|
+
].join("\n");
|
|
22
27
|
let posthogClient;
|
|
23
28
|
let tracerProvider;
|
|
24
29
|
let loggerProvider;
|
|
@@ -26,6 +31,7 @@ let activeConfig;
|
|
|
26
31
|
let telemetryInitialized = false;
|
|
27
32
|
let telemetryStartWarningPrinted = false;
|
|
28
33
|
let telemetryTransportFailed = false;
|
|
34
|
+
let telemetryNoticeThisProcess;
|
|
29
35
|
export function createTelemetryTransportCircuitBreaker(onTransportFailure) {
|
|
30
36
|
let circuitOpen = false;
|
|
31
37
|
return {
|
|
@@ -125,7 +131,7 @@ function disableTelemetryAfterTransportFailure(error) {
|
|
|
125
131
|
process.stderr.write(`[feynman] Telemetry disabled for this session after a transport failure (${error instanceof Error ? error.message : "unknown error"}).\n`);
|
|
126
132
|
}
|
|
127
133
|
}
|
|
128
|
-
function isTelemetryDisabled(env = process.env) {
|
|
134
|
+
export function isTelemetryDisabled(env = process.env) {
|
|
129
135
|
const setting = env.FEYNMAN_TELEMETRY ?? env.FEYNMAN_POSTHOG_TELEMETRY;
|
|
130
136
|
return (setting !== undefined && TELEMETRY_DISABLED_VALUES.has(setting.trim().toLowerCase())) || env.DO_NOT_TRACK === "1";
|
|
131
137
|
}
|
|
@@ -156,6 +162,42 @@ function getAnonymousDistinctId(home = getFeynmanHome()) {
|
|
|
156
162
|
writeFileSync(statePath, JSON.stringify({ ...state, anonymousId }, null, 2) + "\n", "utf8");
|
|
157
163
|
return anonymousId;
|
|
158
164
|
}
|
|
165
|
+
/**
|
|
166
|
+
* Returns the first-run telemetry notice once per Feynman home, and again for
|
|
167
|
+
* the rest of this process so a launch that clears the screen can reprint it.
|
|
168
|
+
* Only interactive terminals get it: scripts and CI (where Windows PowerShell
|
|
169
|
+
* treats any stderr as an error) never see it, and it stays pending for the
|
|
170
|
+
* first interactive run.
|
|
171
|
+
*/
|
|
172
|
+
export function telemetryFirstRunNotice(home = getFeynmanHome(), interactive = process.stderr.isTTY === true) {
|
|
173
|
+
if (telemetryNoticeThisProcess || !activeConfig)
|
|
174
|
+
return telemetryNoticeThisProcess;
|
|
175
|
+
if (!interactive)
|
|
176
|
+
return undefined;
|
|
177
|
+
const statePath = resolve(getFeynmanStateDir(home), TELEMETRY_STATE_FILE);
|
|
178
|
+
const state = readTelemetryState(statePath);
|
|
179
|
+
if (state.noticeShown)
|
|
180
|
+
return undefined;
|
|
181
|
+
try {
|
|
182
|
+
mkdirSync(dirname(statePath), { recursive: true });
|
|
183
|
+
writeFileSync(statePath, JSON.stringify({ ...state, noticeShown: true }, null, 2) + "\n", "utf8");
|
|
184
|
+
}
|
|
185
|
+
catch {
|
|
186
|
+
return undefined;
|
|
187
|
+
}
|
|
188
|
+
telemetryNoticeThisProcess = TELEMETRY_NOTICE;
|
|
189
|
+
return telemetryNoticeThisProcess;
|
|
190
|
+
}
|
|
191
|
+
/** Env the Pi child needs for the research extension to send under the same install id. */
|
|
192
|
+
export function getPostHogChildEnv() {
|
|
193
|
+
if (!activeConfig)
|
|
194
|
+
return {};
|
|
195
|
+
return {
|
|
196
|
+
FEYNMAN_POSTHOG_KEY: activeConfig.projectToken,
|
|
197
|
+
FEYNMAN_POSTHOG_HOST: activeConfig.host,
|
|
198
|
+
FEYNMAN_TELEMETRY_DISTINCT_ID: activeConfig.distinctId,
|
|
199
|
+
};
|
|
200
|
+
}
|
|
159
201
|
export function resolvePostHogTelemetryConfig(options = {}) {
|
|
160
202
|
const env = options.env ?? process.env;
|
|
161
203
|
if (isTelemetryDisabled(env))
|
|
@@ -441,6 +483,7 @@ export async function shutdownPostHogTelemetry() {
|
|
|
441
483
|
activeConfig = undefined;
|
|
442
484
|
telemetryInitialized = false;
|
|
443
485
|
telemetryTransportFailed = false;
|
|
486
|
+
telemetryNoticeThisProcess = undefined;
|
|
444
487
|
await Promise.allSettled([
|
|
445
488
|
client?.shutdown(3000),
|
|
446
489
|
traces?.shutdown(),
|
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
import { readdirSync, statSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
|
|
4
|
+
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
5
|
+
import { PostHog, type PostHogOptions } from "posthog-node";
|
|
6
|
+
|
|
7
|
+
import { readPromptSpecs } from "../../metadata/commands.mjs";
|
|
8
|
+
import { APP_ROOT, FEYNMAN_VERSION } from "./shared.js";
|
|
9
|
+
|
|
10
|
+
// Same opt-out contract as src/telemetry/posthog.ts. The CLI passes the
|
|
11
|
+
// project token, host, and anonymous install id to the Pi child only when its
|
|
12
|
+
// own telemetry is on, so a Pi run outside the CLI sends nothing.
|
|
13
|
+
const TELEMETRY_DISABLED_VALUES = new Set(["0", "false", "no", "off", "disabled"]);
|
|
14
|
+
const FLUSH_TIMEOUT_MS = 2000;
|
|
15
|
+
const OUTPUT_DIRS = ["outputs", "papers"];
|
|
16
|
+
const OUTPUT_SCAN_LIMIT = 5000;
|
|
17
|
+
const SHARED_STATE_KEY = Symbol.for("feynman.research-telemetry");
|
|
18
|
+
|
|
19
|
+
type TelemetryValue = string | number | boolean | undefined;
|
|
20
|
+
export type ResearchTelemetryClient = Pick<PostHog, "capture" | "flush" | "shutdown">;
|
|
21
|
+
export type ResearchTelemetryConfig = { projectToken: string; host: string; distinctId: string };
|
|
22
|
+
|
|
23
|
+
// pi-subagents runs foreground children as in-process sessions that load this
|
|
24
|
+
// extension again with fresh module state, so the owner of the top-level
|
|
25
|
+
// session and the client live on globalThis.
|
|
26
|
+
export type ResearchTelemetrySharedState = {
|
|
27
|
+
client?: ResearchTelemetryClient;
|
|
28
|
+
primary?: object;
|
|
29
|
+
primarySessionId?: string;
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
type WorkflowRun = {
|
|
33
|
+
workflow: string;
|
|
34
|
+
startedAt: number;
|
|
35
|
+
toolCalls: number;
|
|
36
|
+
subagentCalls: number;
|
|
37
|
+
stopReason?: string;
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
export function resolveResearchTelemetryConfig(env: NodeJS.ProcessEnv = process.env): ResearchTelemetryConfig | undefined {
|
|
41
|
+
const setting = env.FEYNMAN_TELEMETRY ?? env.FEYNMAN_POSTHOG_TELEMETRY;
|
|
42
|
+
if ((setting !== undefined && TELEMETRY_DISABLED_VALUES.has(setting.trim().toLowerCase())) || env.DO_NOT_TRACK === "1") {
|
|
43
|
+
return undefined;
|
|
44
|
+
}
|
|
45
|
+
const projectToken = env.FEYNMAN_POSTHOG_KEY?.trim();
|
|
46
|
+
const host = env.FEYNMAN_POSTHOG_HOST?.trim().replace(/\/+$/, "");
|
|
47
|
+
const distinctId = env.FEYNMAN_TELEMETRY_DISTINCT_ID?.trim();
|
|
48
|
+
return projectToken && host && distinctId ? { projectToken, host, distinctId } : undefined;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function createPostHogClient(config: ResearchTelemetryConfig): ResearchTelemetryClient {
|
|
52
|
+
// One attempt per send; the first failure drops the rest of this process's
|
|
53
|
+
// sends silently, matching the CLI's circuit breaker.
|
|
54
|
+
let transportFailed = false;
|
|
55
|
+
const fetchOnce: NonNullable<PostHogOptions["fetch"]> = async (url, options) => {
|
|
56
|
+
if (!transportFailed) {
|
|
57
|
+
try {
|
|
58
|
+
const response = await fetch(url, options as RequestInit);
|
|
59
|
+
if (response.status >= 200 && response.status < 400) return response;
|
|
60
|
+
} catch {}
|
|
61
|
+
transportFailed = true;
|
|
62
|
+
}
|
|
63
|
+
return new Response(null, { status: 204 });
|
|
64
|
+
};
|
|
65
|
+
const client = new PostHog(config.projectToken, {
|
|
66
|
+
host: config.host,
|
|
67
|
+
flushAt: 1,
|
|
68
|
+
flushInterval: 0,
|
|
69
|
+
isServer: false,
|
|
70
|
+
disableGeoip: true,
|
|
71
|
+
fetchRetryCount: 0,
|
|
72
|
+
fetch: fetchOnce,
|
|
73
|
+
});
|
|
74
|
+
client.on("error", () => {});
|
|
75
|
+
return client;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export function workflowName(text: string, workflows: ReadonlySet<string>): string {
|
|
79
|
+
const name = /^\/([A-Za-z0-9_-]+)(?:\s|$)/.exec(text.trimStart())?.[1];
|
|
80
|
+
return name && workflows.has(name) ? name : "chat";
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export function wroteResearchOutput(cwd: string, since: number): boolean {
|
|
84
|
+
let scanned = 0;
|
|
85
|
+
for (const dir of OUTPUT_DIRS) {
|
|
86
|
+
let entries: string[];
|
|
87
|
+
try {
|
|
88
|
+
entries = readdirSync(join(cwd, dir), { recursive: true, encoding: "utf8" });
|
|
89
|
+
} catch {
|
|
90
|
+
continue;
|
|
91
|
+
}
|
|
92
|
+
for (const entry of entries) {
|
|
93
|
+
if (++scanned > OUTPUT_SCAN_LIMIT) return false;
|
|
94
|
+
try {
|
|
95
|
+
const stats = statSync(join(cwd, dir, entry));
|
|
96
|
+
if (stats.isFile() && stats.mtimeMs >= since) return true;
|
|
97
|
+
} catch {}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return false;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function stopStatus(stopReason: string | undefined): "completed" | "error" | "aborted" {
|
|
104
|
+
return stopReason === "error" || stopReason === "aborted" ? stopReason : "completed";
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function readWorkflowNames(): Set<string> {
|
|
108
|
+
try {
|
|
109
|
+
return new Set(readPromptSpecs(APP_ROOT).map((spec) => spec.name));
|
|
110
|
+
} catch {
|
|
111
|
+
return new Set();
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export function registerResearchTelemetry(
|
|
116
|
+
pi: ExtensionAPI,
|
|
117
|
+
options: {
|
|
118
|
+
env?: NodeJS.ProcessEnv;
|
|
119
|
+
createClient?: (config: ResearchTelemetryConfig) => ResearchTelemetryClient;
|
|
120
|
+
shared?: ResearchTelemetrySharedState;
|
|
121
|
+
now?: () => number;
|
|
122
|
+
} = {},
|
|
123
|
+
): void {
|
|
124
|
+
const env = options.env ?? process.env;
|
|
125
|
+
const config = resolveResearchTelemetryConfig(env);
|
|
126
|
+
if (!config) return;
|
|
127
|
+
const now = options.now ?? Date.now;
|
|
128
|
+
const globalSlot = globalThis as { [SHARED_STATE_KEY]?: ResearchTelemetrySharedState };
|
|
129
|
+
const shared = options.shared ?? (globalSlot[SHARED_STATE_KEY] ??= {});
|
|
130
|
+
const workflows = readWorkflowNames();
|
|
131
|
+
const owner = {};
|
|
132
|
+
const baseProperties = {
|
|
133
|
+
app_version: FEYNMAN_VERSION,
|
|
134
|
+
node_version: process.versions.node,
|
|
135
|
+
platform: process.platform,
|
|
136
|
+
arch: process.arch,
|
|
137
|
+
telemetry_source: "feynman",
|
|
138
|
+
$process_person_profile: false,
|
|
139
|
+
};
|
|
140
|
+
|
|
141
|
+
let primary = false;
|
|
142
|
+
let traceId: string | undefined;
|
|
143
|
+
let pendingWorkflow: string | undefined;
|
|
144
|
+
let run: WorkflowRun | undefined;
|
|
145
|
+
let requestStartedAt: number | undefined;
|
|
146
|
+
let httpStatus: number | undefined;
|
|
147
|
+
|
|
148
|
+
const capture = (event: string, properties: Record<string, TelemetryValue>) => {
|
|
149
|
+
try {
|
|
150
|
+
shared.client ??= (options.createClient ?? createPostHogClient)(config);
|
|
151
|
+
shared.client.capture({ distinctId: config.distinctId, event, properties: { ...baseProperties, ...properties } });
|
|
152
|
+
} catch {}
|
|
153
|
+
};
|
|
154
|
+
|
|
155
|
+
const finishRun = (cwd: string, status: "completed" | "error" | "aborted") => {
|
|
156
|
+
if (!run) return;
|
|
157
|
+
const finished = run;
|
|
158
|
+
run = undefined;
|
|
159
|
+
capture("feynman_workflow_completed", {
|
|
160
|
+
workflow: finished.workflow,
|
|
161
|
+
status,
|
|
162
|
+
tool_calls: finished.toolCalls,
|
|
163
|
+
subagent_calls: finished.subagentCalls,
|
|
164
|
+
output_written: wroteResearchOutput(cwd, finished.startedAt),
|
|
165
|
+
duration_ms: now() - finished.startedAt,
|
|
166
|
+
});
|
|
167
|
+
};
|
|
168
|
+
|
|
169
|
+
pi.on("session_start", (event, ctx) => {
|
|
170
|
+
const sessionId = ctx.sessionManager.getSessionId();
|
|
171
|
+
primary = env.PI_SUBAGENT_CHILD !== "1" && shared.primary === undefined;
|
|
172
|
+
if (primary) {
|
|
173
|
+
shared.primary = owner;
|
|
174
|
+
shared.primarySessionId = sessionId;
|
|
175
|
+
}
|
|
176
|
+
traceId = primary ? sessionId : (shared.primarySessionId ?? env.PI_SUBAGENT_PARENT_SESSION ?? sessionId);
|
|
177
|
+
if (!primary) return;
|
|
178
|
+
capture("feynman_session_started", {
|
|
179
|
+
reason: event.reason,
|
|
180
|
+
mode: ctx.mode,
|
|
181
|
+
model_provider: ctx.model?.provider,
|
|
182
|
+
model: ctx.model?.id,
|
|
183
|
+
});
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
pi.on("input", (event) => {
|
|
187
|
+
if (primary && !event.streamingBehavior) pendingWorkflow = workflowName(event.text, workflows);
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
pi.on("agent_start", () => {
|
|
191
|
+
if (!primary || run) return;
|
|
192
|
+
run = { workflow: pendingWorkflow ?? "chat", startedAt: now(), toolCalls: 0, subagentCalls: 0 };
|
|
193
|
+
pendingWorkflow = undefined;
|
|
194
|
+
capture("feynman_workflow_started", { workflow: run.workflow });
|
|
195
|
+
});
|
|
196
|
+
|
|
197
|
+
pi.on("tool_execution_end", (event) => {
|
|
198
|
+
capture("feynman_tool_used", { tool: event.toolName, is_error: event.isError, subagent: !primary });
|
|
199
|
+
if (!run) return;
|
|
200
|
+
run.toolCalls += 1;
|
|
201
|
+
if (event.toolName === "subagent") run.subagentCalls += 1;
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
pi.on("before_provider_request", () => {
|
|
205
|
+
requestStartedAt = now();
|
|
206
|
+
});
|
|
207
|
+
|
|
208
|
+
pi.on("after_provider_response", (event) => {
|
|
209
|
+
httpStatus = event.status;
|
|
210
|
+
});
|
|
211
|
+
|
|
212
|
+
pi.on("message_end", (event) => {
|
|
213
|
+
const message = event.message;
|
|
214
|
+
if (message.role !== "assistant") return;
|
|
215
|
+
// PostHog LLM analytics schema, metadata only: no $ai_input or $ai_output_choices.
|
|
216
|
+
capture("$ai_generation", {
|
|
217
|
+
$ai_trace_id: traceId,
|
|
218
|
+
$ai_model: message.model,
|
|
219
|
+
$ai_provider: message.provider,
|
|
220
|
+
$ai_input_tokens: message.usage.input,
|
|
221
|
+
$ai_output_tokens: message.usage.output,
|
|
222
|
+
$ai_cache_read_input_tokens: message.usage.cacheRead,
|
|
223
|
+
$ai_cache_creation_input_tokens: message.usage.cacheWrite,
|
|
224
|
+
// Pi reports input tokens with cache reads and writes already subtracted.
|
|
225
|
+
$ai_cache_reporting_exclusive: true,
|
|
226
|
+
$ai_latency: requestStartedAt === undefined ? undefined : (now() - requestStartedAt) / 1000,
|
|
227
|
+
$ai_http_status: httpStatus,
|
|
228
|
+
$ai_is_error: message.stopReason === "error",
|
|
229
|
+
$ai_stop_reason: message.stopReason,
|
|
230
|
+
subagent: !primary,
|
|
231
|
+
});
|
|
232
|
+
requestStartedAt = undefined;
|
|
233
|
+
httpStatus = undefined;
|
|
234
|
+
if (run) run.stopReason = message.stopReason;
|
|
235
|
+
});
|
|
236
|
+
|
|
237
|
+
pi.on("agent_settled", (_event, ctx) => {
|
|
238
|
+
finishRun(ctx.cwd, stopStatus(run?.stopReason));
|
|
239
|
+
});
|
|
240
|
+
|
|
241
|
+
pi.on("session_shutdown", async (event, ctx) => {
|
|
242
|
+
finishRun(ctx.cwd, "aborted");
|
|
243
|
+
if (shared.primary === owner) {
|
|
244
|
+
shared.primary = undefined;
|
|
245
|
+
shared.primarySessionId = undefined;
|
|
246
|
+
}
|
|
247
|
+
const client = shared.client;
|
|
248
|
+
if (!client) return;
|
|
249
|
+
const quitting = primary && event.reason === "quit";
|
|
250
|
+
if (quitting) shared.client = undefined;
|
|
251
|
+
let timer: NodeJS.Timeout | undefined;
|
|
252
|
+
const timeout = new Promise<void>((resolve) => {
|
|
253
|
+
timer = setTimeout(resolve, FLUSH_TIMEOUT_MS);
|
|
254
|
+
timer.unref?.();
|
|
255
|
+
});
|
|
256
|
+
try {
|
|
257
|
+
await Promise.race([quitting ? client.shutdown(FLUSH_TIMEOUT_MS) : client.flush(), timeout]);
|
|
258
|
+
} catch {
|
|
259
|
+
} finally {
|
|
260
|
+
clearTimeout(timer);
|
|
261
|
+
}
|
|
262
|
+
});
|
|
263
|
+
}
|
|
@@ -9,6 +9,7 @@ import { registerHuggingFaceTools } from "./research-tools/huggingface.js";
|
|
|
9
9
|
import { registerInitCommand, registerOutputsCommand } from "./research-tools/project.js";
|
|
10
10
|
import { registerServiceTierControls } from "./research-tools/service-tier.js";
|
|
11
11
|
import { registerScienceDatabaseTools } from "./research-tools/science-databases.js";
|
|
12
|
+
import { registerResearchTelemetry } from "./research-tools/telemetry.js";
|
|
12
13
|
|
|
13
14
|
export default function researchTools(pi: ExtensionAPI): void {
|
|
14
15
|
const cache: { agentSummaryPromise?: Promise<{ agents: string[]; chains: string[] }> } = {};
|
|
@@ -27,4 +28,5 @@ export default function researchTools(pi: ExtensionAPI): void {
|
|
|
27
28
|
registerOutputsCommand(pi);
|
|
28
29
|
registerServiceTierControls(pi);
|
|
29
30
|
registerScienceDatabaseTools(pi);
|
|
31
|
+
registerResearchTelemetry(pi);
|
|
30
32
|
}
|