opencandle 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -3
- package/dist/cli.js +36 -0
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +10 -0
- package/dist/config.js +13 -0
- package/dist/config.js.map +1 -1
- package/dist/infra/index.d.ts +0 -1
- package/dist/infra/index.js +0 -1
- package/dist/infra/index.js.map +1 -1
- package/dist/onboarding/connect.d.ts +2 -2
- package/dist/onboarding/connect.js +10 -3
- package/dist/onboarding/connect.js.map +1 -1
- package/dist/onboarding/provider-status.d.ts +48 -0
- package/dist/onboarding/provider-status.js +285 -0
- package/dist/onboarding/provider-status.js.map +1 -0
- package/dist/onboarding/providers.d.ts +85 -8
- package/dist/onboarding/providers.js +87 -9
- package/dist/onboarding/providers.js.map +1 -1
- package/dist/onboarding/state.d.ts +1 -0
- package/dist/onboarding/state.js +5 -0
- package/dist/onboarding/state.js.map +1 -1
- package/dist/onboarding/tool-tags.d.ts +12 -1
- package/dist/onboarding/tool-tags.js +31 -1
- package/dist/onboarding/tool-tags.js.map +1 -1
- package/dist/onboarding/validation.d.ts +2 -2
- package/dist/onboarding/validation.js.map +1 -1
- package/dist/pi/opencandle-extension.js +91 -15
- package/dist/pi/opencandle-extension.js.map +1 -1
- package/dist/pi/tool-adapter.d.ts +4 -1
- package/dist/pi/tool-adapter.js +5 -4
- package/dist/pi/tool-adapter.js.map +1 -1
- package/dist/prompts/context-builder.js +1 -1
- package/dist/prompts/policy-cards.js +1 -1
- package/dist/prompts/policy-cards.js.map +1 -1
- package/dist/providers/external-tool-error.d.ts +10 -0
- package/dist/providers/external-tool-error.js +21 -0
- package/dist/providers/external-tool-error.js.map +1 -0
- package/dist/providers/reddit-cli.d.ts +36 -0
- package/dist/providers/reddit-cli.js +201 -0
- package/dist/providers/reddit-cli.js.map +1 -0
- package/dist/providers/reddit.d.ts +1 -1
- package/dist/providers/reddit.js +7 -35
- package/dist/providers/reddit.js.map +1 -1
- package/dist/providers/twitter-cli.d.ts +40 -0
- package/dist/providers/twitter-cli.js +153 -0
- package/dist/providers/twitter-cli.js.map +1 -0
- package/dist/providers/twitter.d.ts +0 -8
- package/dist/providers/twitter.js +4 -54
- package/dist/providers/twitter.js.map +1 -1
- package/dist/providers/wrap-provider.js +30 -0
- package/dist/providers/wrap-provider.js.map +1 -1
- package/dist/providers/yahoo-finance.js +53 -32
- package/dist/providers/yahoo-finance.js.map +1 -1
- package/dist/routing/planning.d.ts +1 -1
- package/dist/routing/planning.js.map +1 -1
- package/dist/runtime/answer-contracts.d.ts +1 -1
- package/dist/runtime/answer-contracts.js +12 -1
- package/dist/runtime/answer-contracts.js.map +1 -1
- package/dist/runtime/tool-defaults-wrapper.js +6 -2
- package/dist/runtime/tool-defaults-wrapper.js.map +1 -1
- package/dist/sentiment/index.d.ts +1 -0
- package/dist/sentiment/index.js +1 -0
- package/dist/sentiment/index.js.map +1 -1
- package/dist/sentiment/insights.d.ts +17 -0
- package/dist/sentiment/insights.js +206 -0
- package/dist/sentiment/insights.js.map +1 -0
- package/dist/sentiment/pipeline.js +13 -1
- package/dist/sentiment/pipeline.js.map +1 -1
- package/dist/sentiment/scorer.d.ts +2 -0
- package/dist/sentiment/scorer.js +10 -1
- package/dist/sentiment/scorer.js.map +1 -1
- package/dist/sentiment/types.d.ts +2 -0
- package/dist/sentiment/types.js.map +1 -1
- package/dist/system-prompt.js +3 -7
- package/dist/system-prompt.js.map +1 -1
- package/dist/tools/index.d.ts +5 -2
- package/dist/tools/index.js +8 -8
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/sentiment/insight-format.d.ts +2 -0
- package/dist/tools/sentiment/insight-format.js +36 -0
- package/dist/tools/sentiment/insight-format.js.map +1 -0
- package/dist/tools/sentiment/query-match.d.ts +3 -0
- package/dist/tools/sentiment/query-match.js +113 -0
- package/dist/tools/sentiment/query-match.js.map +1 -0
- package/dist/tools/sentiment/reddit-sentiment.d.ts +12 -1
- package/dist/tools/sentiment/reddit-sentiment.js +263 -117
- package/dist/tools/sentiment/reddit-sentiment.js.map +1 -1
- package/dist/tools/sentiment/sentiment-summary.d.ts +9 -1
- package/dist/tools/sentiment/sentiment-summary.js +217 -201
- package/dist/tools/sentiment/sentiment-summary.js.map +1 -1
- package/dist/tools/sentiment/twitter-sentiment.d.ts +11 -1
- package/dist/tools/sentiment/twitter-sentiment.js +187 -64
- package/dist/tools/sentiment/twitter-sentiment.js.map +1 -1
- package/dist/tools/sentiment/web-sentiment.js +4 -0
- package/dist/tools/sentiment/web-sentiment.js.map +1 -1
- package/dist/types/sentiment.d.ts +52 -0
- package/gui/server/invoke-tool.ts +17 -3
- package/gui/server/model-setup.ts +10 -3
- package/gui/server/projector.ts +6 -2
- package/gui/server/server.ts +18 -0
- package/gui/server/tool-metadata.ts +80 -16
- package/gui/server/ws-hub.ts +19 -0
- package/gui/web/dist/assets/CatalogOverlay-CgeY5Pkp.js +1 -0
- package/gui/web/dist/assets/index-C6W_2eAn.js +69 -0
- package/gui/web/dist/assets/{index-2KZtKBmu.css → index-hwbx24a5.css} +1 -1
- package/gui/web/dist/index.html +2 -2
- package/package.json +5 -6
- package/src/cli.ts +41 -0
- package/src/config.ts +27 -0
- package/src/infra/index.ts +0 -1
- package/src/onboarding/connect.ts +20 -4
- package/src/onboarding/provider-status.ts +410 -0
- package/src/onboarding/providers.ts +148 -18
- package/src/onboarding/state.ts +9 -0
- package/src/onboarding/tool-tags.ts +45 -2
- package/src/onboarding/validation.ts +2 -2
- package/src/pi/opencandle-extension.ts +115 -17
- package/src/pi/tool-adapter.ts +14 -4
- package/src/prompts/context-builder.ts +1 -1
- package/src/prompts/policy-cards.ts +1 -1
- package/src/providers/external-tool-error.ts +20 -0
- package/src/providers/reddit-cli.ts +317 -0
- package/src/providers/reddit.ts +7 -63
- package/src/providers/twitter-cli.ts +233 -0
- package/src/providers/twitter.ts +4 -73
- package/src/providers/wrap-provider.ts +34 -0
- package/src/providers/yahoo-finance.ts +65 -32
- package/src/routing/planning.ts +1 -0
- package/src/runtime/answer-contracts.ts +23 -2
- package/src/runtime/tool-defaults-wrapper.ts +12 -2
- package/src/sentiment/index.ts +1 -0
- package/src/sentiment/insights.ts +269 -0
- package/src/sentiment/pipeline.ts +13 -1
- package/src/sentiment/scorer.ts +12 -1
- package/src/sentiment/types.ts +3 -0
- package/src/system-prompt.ts +3 -7
- package/src/tools/index.ts +9 -8
- package/src/tools/sentiment/insight-format.ts +50 -0
- package/src/tools/sentiment/query-match.ts +117 -0
- package/src/tools/sentiment/reddit-sentiment.ts +354 -141
- package/src/tools/sentiment/sentiment-summary.ts +283 -237
- package/src/tools/sentiment/twitter-sentiment.ts +262 -78
- package/src/tools/sentiment/web-sentiment.ts +4 -0
- package/src/types/sentiment.ts +59 -0
- package/dist/infra/browser.d.ts +0 -35
- package/dist/infra/browser.js +0 -105
- package/dist/infra/browser.js.map +0 -1
- package/dist/tools/interaction/twitter-login.d.ts +0 -8
- package/dist/tools/interaction/twitter-login.js +0 -87
- package/dist/tools/interaction/twitter-login.js.map +0 -1
- package/gui/web/dist/assets/CatalogOverlay-eJ2cBk33.js +0 -1
- package/gui/web/dist/assets/index-CveNgtDg.js +0 -69
- package/src/infra/browser.ts +0 -113
- package/src/tools/interaction/twitter-login.ts +0 -105
package/src/providers/reddit.ts
CHANGED
|
@@ -1,53 +1,23 @@
|
|
|
1
1
|
import { cache, STALE_LIMIT, TTL } from "../infra/cache.js";
|
|
2
|
-
import { httpGet } from "../infra/http-client.js";
|
|
3
2
|
import { rateLimiter } from "../infra/rate-limiter.js";
|
|
4
3
|
import { BEARISH_TERMS, BULLISH_TERMS } from "../sentiment/keywords.js";
|
|
5
4
|
import type { RedditSentimentResult } from "../types/sentiment.js";
|
|
6
|
-
|
|
7
|
-
interface RedditListingResponse {
|
|
8
|
-
data: {
|
|
9
|
-
children: Array<{
|
|
10
|
-
data: {
|
|
11
|
-
id: string;
|
|
12
|
-
title: string;
|
|
13
|
-
selftext: string;
|
|
14
|
-
author: string;
|
|
15
|
-
score: number;
|
|
16
|
-
num_comments: number;
|
|
17
|
-
permalink: string;
|
|
18
|
-
created_utc: number;
|
|
19
|
-
};
|
|
20
|
-
}>;
|
|
21
|
-
};
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
const REDDIT_HEADERS = { "User-Agent": "OpenCandle/1.0 (financial analysis agent)" };
|
|
5
|
+
import { listSubredditPosts, readRedditPost, searchRedditPosts } from "./reddit-cli.js";
|
|
25
6
|
|
|
26
7
|
export async function getSubredditPosts(
|
|
27
8
|
subreddit: string,
|
|
28
9
|
limit: number = 25,
|
|
10
|
+
query?: string,
|
|
29
11
|
): Promise<RedditSentimentResult> {
|
|
30
|
-
const cacheKey = `reddit:${subreddit}:${limit}`;
|
|
12
|
+
const cacheKey = `reddit:${subreddit}:${query ?? "hot"}:${limit}`;
|
|
31
13
|
const cached = cache.get<RedditSentimentResult>(cacheKey);
|
|
32
14
|
if (cached) return cached;
|
|
33
15
|
|
|
34
16
|
try {
|
|
35
17
|
await rateLimiter.acquire("reddit");
|
|
36
|
-
const
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
});
|
|
40
|
-
|
|
41
|
-
const posts = data.data.children.map((child) => ({
|
|
42
|
-
id: child.data.id,
|
|
43
|
-
title: child.data.title,
|
|
44
|
-
selftext: child.data.selftext ?? "",
|
|
45
|
-
author: child.data.author ?? "unknown",
|
|
46
|
-
score: child.data.score,
|
|
47
|
-
comments: child.data.num_comments,
|
|
48
|
-
url: `https://reddit.com${child.data.permalink}`,
|
|
49
|
-
created: new Date(child.data.created_utc * 1000).toISOString(),
|
|
50
|
-
}));
|
|
18
|
+
const posts = query
|
|
19
|
+
? await searchRedditPosts(query, { subreddit, limit })
|
|
20
|
+
: await listSubredditPosts(subreddit, { limit });
|
|
51
21
|
|
|
52
22
|
// Extract ticker-like mentions ($AAPL, $TSLA, etc.)
|
|
53
23
|
const tickerRegex = /\$([A-Z]{1,5})\b/g;
|
|
@@ -107,33 +77,7 @@ export async function getPostComments(
|
|
|
107
77
|
if (cached) return cached;
|
|
108
78
|
|
|
109
79
|
await rateLimiter.acquire("reddit_comments");
|
|
110
|
-
const
|
|
111
|
-
const data = await httpGet<
|
|
112
|
-
Array<{
|
|
113
|
-
data: {
|
|
114
|
-
children: Array<{
|
|
115
|
-
kind: string;
|
|
116
|
-
data: { id: string; body?: string; author?: string; score?: number; permalink?: string };
|
|
117
|
-
}>;
|
|
118
|
-
};
|
|
119
|
-
}>
|
|
120
|
-
>(url, {
|
|
121
|
-
headers: REDDIT_HEADERS,
|
|
122
|
-
});
|
|
123
|
-
|
|
124
|
-
// Comments are in the second listing element
|
|
125
|
-
const commentListing = data[1]?.data?.children ?? [];
|
|
126
|
-
const comments: RedditComment[] = commentListing
|
|
127
|
-
.filter((c) => c.kind === "t1" && c.data.body)
|
|
128
|
-
.sort((a, b) => (b.data.score ?? 0) - (a.data.score ?? 0))
|
|
129
|
-
.slice(0, limit)
|
|
130
|
-
.map((c) => ({
|
|
131
|
-
id: c.data.id,
|
|
132
|
-
body: c.data.body!,
|
|
133
|
-
author: c.data.author ?? "unknown",
|
|
134
|
-
score: c.data.score ?? 0,
|
|
135
|
-
permalink: `https://reddit.com${c.data.permalink ?? ""}`,
|
|
136
|
-
}));
|
|
80
|
+
const { comments } = await readRedditPost(postId, { limit });
|
|
137
81
|
|
|
138
82
|
cache.set(cacheKey, comments, COMMENT_TTL);
|
|
139
83
|
return comments;
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import type { TwitterTweet } from "../types/sentiment.js";
|
|
3
|
+
import { ExternalToolError, ExternalToolNotInstalled } from "./external-tool-error.js";
|
|
4
|
+
|
|
5
|
+
const TWITTER_CLI_BINARY = "twitter";
|
|
6
|
+
const TWITTER_CLI_TOOL_NAME = "twitter-cli";
|
|
7
|
+
const TWITTER_CLI_INSTALL_CMD = "uv tool install twitter-cli";
|
|
8
|
+
const COMMAND_TIMEOUT_MS = 20_000;
|
|
9
|
+
const MAX_OUTPUT_CHARS = 2_000_000;
|
|
10
|
+
|
|
11
|
+
export interface RawTweet {
|
|
12
|
+
readonly id?: string;
|
|
13
|
+
readonly text?: string;
|
|
14
|
+
readonly author?: {
|
|
15
|
+
readonly username?: string;
|
|
16
|
+
readonly screenName?: string;
|
|
17
|
+
readonly name?: string;
|
|
18
|
+
};
|
|
19
|
+
readonly username?: string;
|
|
20
|
+
readonly url?: string;
|
|
21
|
+
readonly permanentUrl?: string;
|
|
22
|
+
readonly createdAt?: string | number;
|
|
23
|
+
readonly created_at?: string | number;
|
|
24
|
+
readonly likeCount?: number;
|
|
25
|
+
readonly likes?: number;
|
|
26
|
+
readonly retweetCount?: number;
|
|
27
|
+
readonly retweets?: number;
|
|
28
|
+
readonly replyCount?: number;
|
|
29
|
+
readonly replies?: number;
|
|
30
|
+
readonly viewCount?: number | null;
|
|
31
|
+
readonly views?: number | null;
|
|
32
|
+
readonly metrics?: {
|
|
33
|
+
readonly likes?: number;
|
|
34
|
+
readonly retweets?: number;
|
|
35
|
+
readonly replies?: number;
|
|
36
|
+
readonly views?: number | null;
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
interface TwitterCliEnvelope<T> {
|
|
41
|
+
readonly ok: boolean;
|
|
42
|
+
readonly schema_version: string;
|
|
43
|
+
readonly data: T;
|
|
44
|
+
readonly error?: {
|
|
45
|
+
readonly code?: string;
|
|
46
|
+
readonly message?: string;
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
interface CommandResult {
|
|
51
|
+
readonly code: number | null;
|
|
52
|
+
readonly stdout: string;
|
|
53
|
+
readonly stderr: string;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
type TwitterCliCommandRunner = (command: string, args: readonly string[]) => Promise<CommandResult>;
|
|
57
|
+
|
|
58
|
+
let commandRunner: TwitterCliCommandRunner = runCommand;
|
|
59
|
+
|
|
60
|
+
export function setTwitterCliCommandRunnerForTests(runner: TwitterCliCommandRunner): void {
|
|
61
|
+
commandRunner = runner;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export function resetTwitterCliCommandRunnerForTests(): void {
|
|
65
|
+
commandRunner = runCommand;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export async function searchTweets(query: string, max = 20): Promise<TwitterTweet[]> {
|
|
69
|
+
const envelope = await runTwitterCli<TwitterCliEnvelope<RawTweet[]>>([
|
|
70
|
+
"search",
|
|
71
|
+
query,
|
|
72
|
+
"--max",
|
|
73
|
+
String(max),
|
|
74
|
+
"-t",
|
|
75
|
+
"Latest",
|
|
76
|
+
"--json",
|
|
77
|
+
]);
|
|
78
|
+
|
|
79
|
+
if (!envelope.ok) {
|
|
80
|
+
throw new ExternalToolError(
|
|
81
|
+
TWITTER_CLI_TOOL_NAME,
|
|
82
|
+
redactSensitiveOutput(envelope.error?.message ?? "twitter-cli returned an error"),
|
|
83
|
+
envelope.error?.code,
|
|
84
|
+
);
|
|
85
|
+
}
|
|
86
|
+
if (!Array.isArray(envelope.data)) {
|
|
87
|
+
throw new ExternalToolError(TWITTER_CLI_TOOL_NAME, "twitter-cli returned invalid tweet data");
|
|
88
|
+
}
|
|
89
|
+
return envelope.data.map(adaptRawTweet);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
async function runTwitterCli<T>(args: readonly string[]): Promise<T> {
|
|
93
|
+
let result: CommandResult;
|
|
94
|
+
try {
|
|
95
|
+
result = await commandRunner(TWITTER_CLI_BINARY, args);
|
|
96
|
+
} catch (err) {
|
|
97
|
+
const nodeError = err as NodeJS.ErrnoException;
|
|
98
|
+
if (nodeError.code === "ENOENT") {
|
|
99
|
+
throw new ExternalToolNotInstalled(TWITTER_CLI_TOOL_NAME, TWITTER_CLI_INSTALL_CMD);
|
|
100
|
+
}
|
|
101
|
+
throw new ExternalToolError(
|
|
102
|
+
TWITTER_CLI_TOOL_NAME,
|
|
103
|
+
redactSensitiveOutput(err instanceof Error ? err.message : String(err)),
|
|
104
|
+
);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
if (result.code !== 0) {
|
|
108
|
+
const envelopeError = parseCliErrorEnvelope(result.stdout);
|
|
109
|
+
if (envelopeError) {
|
|
110
|
+
throw new ExternalToolError(
|
|
111
|
+
TWITTER_CLI_TOOL_NAME,
|
|
112
|
+
redactSensitiveOutput(envelopeError.message),
|
|
113
|
+
envelopeError.code,
|
|
114
|
+
);
|
|
115
|
+
}
|
|
116
|
+
throw new ExternalToolError(
|
|
117
|
+
TWITTER_CLI_TOOL_NAME,
|
|
118
|
+
redactSensitiveOutput(result.stderr.trim() || `twitter-cli exited with code ${result.code}`),
|
|
119
|
+
);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
try {
|
|
123
|
+
return JSON.parse(result.stdout) as T;
|
|
124
|
+
} catch {
|
|
125
|
+
throw new ExternalToolError(
|
|
126
|
+
TWITTER_CLI_TOOL_NAME,
|
|
127
|
+
`twitter-cli returned non-JSON output: ${redactSensitiveOutput(result.stdout.slice(0, 200))}`,
|
|
128
|
+
);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function parseCliErrorEnvelope(stdout: string): { code?: string; message: string } | null {
|
|
133
|
+
try {
|
|
134
|
+
const parsed = JSON.parse(stdout) as {
|
|
135
|
+
ok?: unknown;
|
|
136
|
+
error?: { code?: unknown; message?: unknown };
|
|
137
|
+
};
|
|
138
|
+
if (parsed.ok !== false || typeof parsed.error?.message !== "string") return null;
|
|
139
|
+
return {
|
|
140
|
+
code: typeof parsed.error.code === "string" ? parsed.error.code : undefined,
|
|
141
|
+
message: parsed.error.message,
|
|
142
|
+
};
|
|
143
|
+
} catch {
|
|
144
|
+
return null;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function adaptRawTweet(raw: RawTweet): TwitterTweet {
|
|
149
|
+
return {
|
|
150
|
+
id: stringValue(raw.id),
|
|
151
|
+
text: stringValue(raw.text).slice(0, 280),
|
|
152
|
+
author:
|
|
153
|
+
stringValue(raw.author?.username) ||
|
|
154
|
+
stringValue(raw.author?.screenName) ||
|
|
155
|
+
stringValue(raw.username) ||
|
|
156
|
+
"unknown",
|
|
157
|
+
likes: numberValue(raw.metrics?.likes ?? raw.likeCount ?? raw.likes),
|
|
158
|
+
retweets: numberValue(raw.metrics?.retweets ?? raw.retweetCount ?? raw.retweets),
|
|
159
|
+
replies: numberValue(raw.metrics?.replies ?? raw.replyCount ?? raw.replies),
|
|
160
|
+
views: nullableNumberValue(raw.metrics?.views ?? raw.viewCount ?? raw.views),
|
|
161
|
+
url: stringValue(raw.url ?? raw.permanentUrl),
|
|
162
|
+
created: normalizeCreatedAt(raw.createdAt ?? raw.created_at),
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
function normalizeCreatedAt(value: string | number | undefined): string {
|
|
167
|
+
if (typeof value === "number") {
|
|
168
|
+
const millis = value > 1_000_000_000_000 ? value : value * 1000;
|
|
169
|
+
return new Date(millis).toISOString();
|
|
170
|
+
}
|
|
171
|
+
if (typeof value === "string" && value.length > 0) {
|
|
172
|
+
const millis = Date.parse(value);
|
|
173
|
+
if (!Number.isNaN(millis)) return new Date(millis).toISOString();
|
|
174
|
+
}
|
|
175
|
+
return new Date(0).toISOString();
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
function stringValue(value: unknown): string {
|
|
179
|
+
return typeof value === "string" ? value : "";
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function numberValue(value: unknown): number {
|
|
183
|
+
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
function nullableNumberValue(value: unknown): number | null {
|
|
187
|
+
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
export function redactSensitiveOutput(input: string): string {
|
|
191
|
+
const xCookieNames =
|
|
192
|
+
"auth_token|ct0|twid|kdt|guest_id|guest_id_ads|guest_id_marketing|personalization_id";
|
|
193
|
+
return input
|
|
194
|
+
.slice(0, MAX_OUTPUT_CHARS)
|
|
195
|
+
.replace(/\b(cookie|set-cookie)\s*:\s*[^\r\n]+/gi, "$1: [redacted]")
|
|
196
|
+
.replace(new RegExp(`\\b(${xCookieNames})\\b\\s*[:=]\\s*[^;\\s,)]+`, "gi"), "$1=[redacted]")
|
|
197
|
+
.replace(new RegExp(`\\b(${xCookieNames})=([^;\\s,)]+)`, "gi"), "$1=[redacted]");
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
function runCommand(command: string, args: readonly string[]): Promise<CommandResult> {
|
|
201
|
+
return new Promise((resolve, reject) => {
|
|
202
|
+
const child = spawn(command, [...args], { stdio: ["ignore", "pipe", "pipe"] });
|
|
203
|
+
let stdout = "";
|
|
204
|
+
let stderr = "";
|
|
205
|
+
let settled = false;
|
|
206
|
+
|
|
207
|
+
const timeout = setTimeout(() => {
|
|
208
|
+
if (settled) return;
|
|
209
|
+
settled = true;
|
|
210
|
+
child.kill("SIGTERM");
|
|
211
|
+
reject(new Error(`${command} timed out after ${COMMAND_TIMEOUT_MS}ms`));
|
|
212
|
+
}, COMMAND_TIMEOUT_MS);
|
|
213
|
+
|
|
214
|
+
child.stdout.on("data", (chunk: Buffer) => {
|
|
215
|
+
stdout = (stdout + chunk.toString("utf8")).slice(0, MAX_OUTPUT_CHARS);
|
|
216
|
+
});
|
|
217
|
+
child.stderr.on("data", (chunk: Buffer) => {
|
|
218
|
+
stderr = (stderr + chunk.toString("utf8")).slice(0, MAX_OUTPUT_CHARS);
|
|
219
|
+
});
|
|
220
|
+
child.on("error", (err) => {
|
|
221
|
+
if (settled) return;
|
|
222
|
+
settled = true;
|
|
223
|
+
clearTimeout(timeout);
|
|
224
|
+
reject(err);
|
|
225
|
+
});
|
|
226
|
+
child.on("close", (code) => {
|
|
227
|
+
if (settled) return;
|
|
228
|
+
settled = true;
|
|
229
|
+
clearTimeout(timeout);
|
|
230
|
+
resolve({ code, stdout, stderr });
|
|
231
|
+
});
|
|
232
|
+
});
|
|
233
|
+
}
|
package/src/providers/twitter.ts
CHANGED
|
@@ -1,37 +1,7 @@
|
|
|
1
|
-
import { existsSync } from "node:fs";
|
|
2
|
-
import { join } from "node:path";
|
|
3
|
-
import { Scraper, SearchMode } from "@the-convocation/twitter-scraper";
|
|
4
|
-
import Database from "better-sqlite3";
|
|
5
1
|
import { cache, STALE_LIMIT, TTL } from "../infra/cache.js";
|
|
6
|
-
import { getBrowserProfileDir } from "../infra/opencandle-paths.js";
|
|
7
2
|
import { rateLimiter } from "../infra/rate-limiter.js";
|
|
8
3
|
import type { TwitterSentimentResult, TwitterTweet } from "../types/sentiment.js";
|
|
9
|
-
|
|
10
|
-
// ── Cookie extraction ────────────────────────────────────
|
|
11
|
-
|
|
12
|
-
interface FirefoxCookie {
|
|
13
|
-
name: string;
|
|
14
|
-
value: string;
|
|
15
|
-
domain: string;
|
|
16
|
-
path: string;
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
export function readTwitterCookies(profileDir: string): FirefoxCookie[] {
|
|
20
|
-
const dbPath = join(profileDir, "cookies.sqlite");
|
|
21
|
-
if (!existsSync(dbPath)) return [];
|
|
22
|
-
|
|
23
|
-
const db = new Database(dbPath, { readonly: true });
|
|
24
|
-
try {
|
|
25
|
-
const rows = db
|
|
26
|
-
.prepare(
|
|
27
|
-
`SELECT name, value, host AS domain, path FROM moz_cookies WHERE host LIKE ? OR host LIKE ?`,
|
|
28
|
-
)
|
|
29
|
-
.all("%x.com%", "%twitter.com%") as FirefoxCookie[];
|
|
30
|
-
return rows;
|
|
31
|
-
} finally {
|
|
32
|
-
db.close();
|
|
33
|
-
}
|
|
34
|
-
}
|
|
4
|
+
import { searchTweets } from "./twitter-cli.js";
|
|
35
5
|
|
|
36
6
|
// ── Sentiment scoring ────────────────────────────────────
|
|
37
7
|
|
|
@@ -87,49 +57,10 @@ export async function getTwitterSentiment(
|
|
|
87
57
|
await rateLimiter.acquire("twitter");
|
|
88
58
|
|
|
89
59
|
try {
|
|
90
|
-
const profileDir = getBrowserProfileDir();
|
|
91
|
-
const cookies = readTwitterCookies(profileDir);
|
|
92
|
-
|
|
93
|
-
const authToken = cookies.find((c) => c.name === "auth_token");
|
|
94
|
-
const ct0 = cookies.find((c) => c.name === "ct0");
|
|
95
|
-
|
|
96
|
-
if (!authToken || !ct0) {
|
|
97
|
-
throw new Error("No Twitter session found.");
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
const scraper = new Scraper();
|
|
101
|
-
const cookieStrings = cookies.map(
|
|
102
|
-
(c) => `${c.name}=${c.value}; Domain=${c.domain}; Path=${c.path}`,
|
|
103
|
-
);
|
|
104
|
-
await scraper.setCookies(cookieStrings);
|
|
105
|
-
|
|
106
|
-
const loggedIn = await scraper.isLoggedIn();
|
|
107
|
-
if (!loggedIn) {
|
|
108
|
-
throw new Error("Twitter session expired.");
|
|
109
|
-
}
|
|
110
|
-
|
|
111
60
|
const cutoff = new Date(Date.now() - hours * 3_600_000);
|
|
112
|
-
const tweets: TwitterTweet[] =
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
for await (const tweet of results) {
|
|
116
|
-
const created = tweet.timeParsed ?? new Date(0);
|
|
117
|
-
if (created < cutoff) continue;
|
|
118
|
-
|
|
119
|
-
tweets.push({
|
|
120
|
-
id: tweet.id ?? "",
|
|
121
|
-
text: tweet.text?.slice(0, 280) ?? "",
|
|
122
|
-
author: tweet.username ?? "unknown",
|
|
123
|
-
likes: tweet.likes ?? 0,
|
|
124
|
-
retweets: tweet.retweets ?? 0,
|
|
125
|
-
replies: tweet.replies ?? 0,
|
|
126
|
-
views: tweet.views ?? null,
|
|
127
|
-
url: tweet.permanentUrl ?? "",
|
|
128
|
-
created: created.toISOString(),
|
|
129
|
-
});
|
|
130
|
-
|
|
131
|
-
if (tweets.length >= limit) break;
|
|
132
|
-
}
|
|
61
|
+
const tweets: TwitterTweet[] = (await searchTweets(normalizedQuery, limit))
|
|
62
|
+
.filter((tweet) => new Date(tweet.created) >= cutoff)
|
|
63
|
+
.slice(0, limit);
|
|
133
64
|
|
|
134
65
|
// Extract co-mentioned cashtags
|
|
135
66
|
const tickerRegex = /\$([A-Z]{1,5})\b/g;
|
|
@@ -2,6 +2,7 @@ import { runWithStaleMetadata } from "../infra/cache.js";
|
|
|
2
2
|
import type { ProviderResult } from "../runtime/evidence.js";
|
|
3
3
|
import { getProviderTracker } from "../runtime/run-context.js";
|
|
4
4
|
import { InvalidSymbolError } from "./errors.js";
|
|
5
|
+
import { ExternalToolError, ExternalToolNotInstalled } from "./external-tool-error.js";
|
|
5
6
|
import { ProviderCredentialError } from "./provider-credential-error.js";
|
|
6
7
|
|
|
7
8
|
/**
|
|
@@ -51,6 +52,13 @@ export async function wrapProvider<T>(
|
|
|
51
52
|
if (error instanceof ProviderCredentialError) {
|
|
52
53
|
throw error;
|
|
53
54
|
}
|
|
55
|
+
if (isExternalToolSetupError(error)) {
|
|
56
|
+
return {
|
|
57
|
+
status: "unavailable",
|
|
58
|
+
reason: error.message,
|
|
59
|
+
provider: providerId,
|
|
60
|
+
};
|
|
61
|
+
}
|
|
54
62
|
if (error instanceof InvalidSymbolError) {
|
|
55
63
|
return {
|
|
56
64
|
status: "unavailable",
|
|
@@ -67,3 +75,29 @@ export async function wrapProvider<T>(
|
|
|
67
75
|
};
|
|
68
76
|
}
|
|
69
77
|
}
|
|
78
|
+
|
|
79
|
+
function isExternalToolSetupError(
|
|
80
|
+
error: unknown,
|
|
81
|
+
): error is ExternalToolError | ExternalToolNotInstalled {
|
|
82
|
+
if (error instanceof ExternalToolNotInstalled) return true;
|
|
83
|
+
if (!(error instanceof ExternalToolError)) return false;
|
|
84
|
+
|
|
85
|
+
const code = error.code?.toLowerCase() ?? "";
|
|
86
|
+
const message = error.message.toLowerCase();
|
|
87
|
+
return (
|
|
88
|
+
code.includes("auth") ||
|
|
89
|
+
code.includes("session") ||
|
|
90
|
+
code.includes("unauthorized") ||
|
|
91
|
+
code.includes("expired") ||
|
|
92
|
+
message.includes("no twitter cookies") ||
|
|
93
|
+
message.includes("no twitter session") ||
|
|
94
|
+
message.includes("no reddit cookies") ||
|
|
95
|
+
message.includes("session missing") ||
|
|
96
|
+
message.includes("not authenticated") ||
|
|
97
|
+
message.includes("no cookies") ||
|
|
98
|
+
message.includes("rdt login") ||
|
|
99
|
+
message.includes("401") ||
|
|
100
|
+
message.includes("unauthorized") ||
|
|
101
|
+
message.includes("expired")
|
|
102
|
+
);
|
|
103
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import
|
|
1
|
+
import YahooFinance from "yahoo-finance2";
|
|
2
|
+
import type { OptionsResult as YahooFinance2OptionsResult } from "yahoo-finance2/modules/options";
|
|
2
3
|
import { cache, STALE_LIMIT, TTL } from "../infra/cache.js";
|
|
3
4
|
import { HttpError, httpGet } from "../infra/http-client.js";
|
|
4
5
|
import { rateLimiter } from "../infra/rate-limiter.js";
|
|
@@ -17,6 +18,15 @@ const BASE_URL = "https://query1.finance.yahoo.com/v8/finance/chart";
|
|
|
17
18
|
const QUOTE_SUMMARY_URL = "https://query1.finance.yahoo.com/v10/finance/quoteSummary";
|
|
18
19
|
const STALE_QUOTE_MAX_RETRY_AFTER_MS = 1_000;
|
|
19
20
|
|
|
21
|
+
let yahooFinance2Client: InstanceType<typeof YahooFinance> | undefined;
|
|
22
|
+
|
|
23
|
+
function getYahooFinance2Client(): InstanceType<typeof YahooFinance> {
|
|
24
|
+
yahooFinance2Client ??= new YahooFinance({
|
|
25
|
+
suppressNotices: ["yahooSurvey", "ripHistorical"],
|
|
26
|
+
});
|
|
27
|
+
return yahooFinance2Client;
|
|
28
|
+
}
|
|
29
|
+
|
|
20
30
|
type YahooNumber = number | { raw?: number; fmt?: string };
|
|
21
31
|
|
|
22
32
|
interface YahooChartResponse {
|
|
@@ -410,9 +420,9 @@ export async function getOptionsChain(symbol: string, expiration?: number): Prom
|
|
|
410
420
|
if (!res?.ok) {
|
|
411
421
|
let browserError: unknown;
|
|
412
422
|
try {
|
|
413
|
-
const
|
|
414
|
-
if (
|
|
415
|
-
const chain = parseOptionsResponse(
|
|
423
|
+
const fallbackData = await fetchOptionsViaYahooFinance2(symbol, expiration);
|
|
424
|
+
if (fallbackData) {
|
|
425
|
+
const chain = parseOptionsResponse(fallbackData);
|
|
416
426
|
cache.set(cacheKey, chain, TTL.OPTIONS_CHAIN);
|
|
417
427
|
return chain;
|
|
418
428
|
}
|
|
@@ -425,14 +435,14 @@ export async function getOptionsChain(symbol: string, expiration?: number): Prom
|
|
|
425
435
|
if (res) {
|
|
426
436
|
const message = `Yahoo Finance options: HTTP ${res.status}`;
|
|
427
437
|
if (browserError instanceof Error) {
|
|
428
|
-
throw new Error(`${message};
|
|
438
|
+
throw new Error(`${message}; yahoo-finance2 fallback failed: ${browserError.message}`);
|
|
429
439
|
}
|
|
430
440
|
throw new Error(message);
|
|
431
441
|
}
|
|
432
442
|
if (browserError instanceof Error) {
|
|
433
443
|
const message =
|
|
434
444
|
fetchError instanceof Error ? fetchError.message : "Yahoo Finance options: fetch failed";
|
|
435
|
-
throw new Error(`${message};
|
|
445
|
+
throw new Error(`${message}; yahoo-finance2 fallback failed: ${browserError.message}`);
|
|
436
446
|
}
|
|
437
447
|
throw fetchError instanceof Error
|
|
438
448
|
? fetchError
|
|
@@ -602,38 +612,61 @@ function parseOptionsResponse(data: YahooOptionsResponse): OptionsChain {
|
|
|
602
612
|
}
|
|
603
613
|
|
|
604
614
|
/**
|
|
605
|
-
* Fallback: fetch options data via
|
|
606
|
-
* Bypasses Yahoo's TLS fingerprinting and rate limiting.
|
|
615
|
+
* Fallback: fetch options data via yahoo-finance2 when the raw Yahoo path is blocked.
|
|
607
616
|
*/
|
|
608
|
-
async function
|
|
617
|
+
async function fetchOptionsViaYahooFinance2(
|
|
609
618
|
symbol: string,
|
|
610
619
|
expiration?: number,
|
|
611
620
|
): Promise<YahooOptionsResponse | null> {
|
|
612
621
|
try {
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
return await StealthBrowser.run(async (page) => {
|
|
619
|
-
await page.goto("https://query2.finance.yahoo.com/v1/test/getcrumb", {
|
|
620
|
-
waitUntil: "domcontentloaded",
|
|
621
|
-
timeout: 15000,
|
|
622
|
-
});
|
|
623
|
-
const crumb = (await page.locator("body").innerText()).trim();
|
|
624
|
-
if (!crumb) return null;
|
|
625
|
-
|
|
626
|
-
const url = `https://query1.finance.yahoo.com/v7/finance/options/${encodeURIComponent(symbol)}?crumb=${encodeURIComponent(crumb)}${dateParam}`;
|
|
627
|
-
const response = await page.goto(url, {
|
|
628
|
-
waitUntil: "domcontentloaded",
|
|
629
|
-
timeout: 15000,
|
|
630
|
-
});
|
|
631
|
-
if (!response?.ok()) return null;
|
|
632
|
-
|
|
633
|
-
const text = (await page.locator("body").innerText()).trim();
|
|
634
|
-
return JSON.parse(text) as YahooOptionsResponse;
|
|
635
|
-
});
|
|
622
|
+
const result = await getYahooFinance2Client().options(
|
|
623
|
+
symbol,
|
|
624
|
+
expiration ? { date: new Date(expiration * 1000) } : undefined,
|
|
625
|
+
);
|
|
626
|
+
return normalizeYahooFinance2OptionsResponse(result);
|
|
636
627
|
} catch (error) {
|
|
637
628
|
throw error instanceof Error ? error : new Error(String(error));
|
|
638
629
|
}
|
|
639
630
|
}
|
|
631
|
+
|
|
632
|
+
function normalizeYahooFinance2OptionsResponse(
|
|
633
|
+
data: YahooFinance2OptionsResult | YahooOptionsResponse,
|
|
634
|
+
): YahooOptionsResponse {
|
|
635
|
+
if ("optionChain" in data) return data as YahooOptionsResponse;
|
|
636
|
+
|
|
637
|
+
const options = data.options.map((option) => ({
|
|
638
|
+
expirationDate: toYahooUnixSeconds(option.expirationDate),
|
|
639
|
+
calls: option.calls,
|
|
640
|
+
puts: option.puts,
|
|
641
|
+
}));
|
|
642
|
+
const strikes = [
|
|
643
|
+
...new Set(
|
|
644
|
+
options
|
|
645
|
+
.flatMap((option) => [...option.calls, ...option.puts])
|
|
646
|
+
.map((contract) => Number((contract as { strike?: unknown }).strike))
|
|
647
|
+
.filter((strike) => Number.isFinite(strike)),
|
|
648
|
+
),
|
|
649
|
+
].sort((a, b) => a - b);
|
|
650
|
+
|
|
651
|
+
return {
|
|
652
|
+
optionChain: {
|
|
653
|
+
result: [
|
|
654
|
+
{
|
|
655
|
+
underlyingSymbol: data.underlyingSymbol,
|
|
656
|
+
expirationDates: data.expirationDates.map(toYahooUnixSeconds),
|
|
657
|
+
strikes,
|
|
658
|
+
quote: data.quote as Record<string, any>,
|
|
659
|
+
options,
|
|
660
|
+
},
|
|
661
|
+
],
|
|
662
|
+
},
|
|
663
|
+
};
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
function toYahooUnixSeconds(value: Date | number | string): number {
|
|
667
|
+
if (value instanceof Date) return Math.floor(value.getTime() / 1000);
|
|
668
|
+
if (typeof value === "number") {
|
|
669
|
+
return value > 1_000_000_000_000 ? Math.floor(value / 1000) : value;
|
|
670
|
+
}
|
|
671
|
+
return Math.floor(new Date(value).getTime() / 1000);
|
|
672
|
+
}
|
package/src/routing/planning.ts
CHANGED
|
@@ -18,7 +18,9 @@ export type FinalAnswerField =
|
|
|
18
18
|
| "freshness_disclosure"
|
|
19
19
|
| "data_gap_disclosure"
|
|
20
20
|
| "risk_downside"
|
|
21
|
-
| "source_coverage"
|
|
21
|
+
| "source_coverage"
|
|
22
|
+
| "sentiment_rationale"
|
|
23
|
+
| "confidence_or_caveats";
|
|
22
24
|
|
|
23
25
|
export type FrameworkFallbackMode =
|
|
24
26
|
| "not_allowed"
|
|
@@ -342,7 +344,12 @@ export const ANSWER_CONTRACT_REGISTRY: Record<AnswerContractId, AnswerContractDe
|
|
|
342
344
|
commitmentMode: "framework",
|
|
343
345
|
implemented: true,
|
|
344
346
|
requiredEvidenceTypes: [],
|
|
345
|
-
requiredFinalFields: [
|
|
347
|
+
requiredFinalFields: [
|
|
348
|
+
"source_coverage",
|
|
349
|
+
"sentiment_rationale",
|
|
350
|
+
"confidence_or_caveats",
|
|
351
|
+
"data_gap_disclosure",
|
|
352
|
+
],
|
|
346
353
|
requiresFreshness: false,
|
|
347
354
|
requiresDataGapDisclosure: true,
|
|
348
355
|
requiresRiskDownside: false,
|
|
@@ -408,6 +415,7 @@ export function runStructuredChecks(input: StructuredCheckInput): StructuredChec
|
|
|
408
415
|
checkFreshness(input.contract, input.finalAnswerMetadata),
|
|
409
416
|
checkDataGapDisclosure(input.contract, input.evidenceRecords, input.finalAnswerMetadata),
|
|
410
417
|
checkCommitmentMode(input.contract, input.finalAnswerMetadata),
|
|
418
|
+
checkRequiredFinalFields(input.contract, input.finalAnswerMetadata),
|
|
411
419
|
checkSourceCoverage(input.contract, input.finalAnswerMetadata),
|
|
412
420
|
checkCapabilityGapDisclosure(input.contract, input.evidenceRecords, input.finalAnswerMetadata),
|
|
413
421
|
...semanticChecks(requestedChecks, input.answerText),
|
|
@@ -620,6 +628,19 @@ function checkCommitmentMode(
|
|
|
620
628
|
);
|
|
621
629
|
}
|
|
622
630
|
|
|
631
|
+
function checkRequiredFinalFields(
|
|
632
|
+
contract: AnswerContractDefinition,
|
|
633
|
+
metadata: FinalAnswerMetadata,
|
|
634
|
+
): StructuredCheckResult {
|
|
635
|
+
const fields = new Set(metadata.finalFields);
|
|
636
|
+
const missing = contract.requiredFinalFields.filter((field) => !fields.has(field));
|
|
637
|
+
return structuredResult(
|
|
638
|
+
"required_final_fields_present",
|
|
639
|
+
missing.length === 0,
|
|
640
|
+
missing.length > 0 ? `Missing required final fields: ${missing.join(", ")}` : undefined,
|
|
641
|
+
);
|
|
642
|
+
}
|
|
643
|
+
|
|
623
644
|
function checkSourceCoverage(
|
|
624
645
|
contract: AnswerContractDefinition,
|
|
625
646
|
metadata: FinalAnswerMetadata,
|