openplanter 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +210 -0
- package/dist/builder.d.ts +11 -0
- package/dist/builder.d.ts.map +1 -0
- package/dist/builder.js +179 -0
- package/dist/builder.js.map +1 -0
- package/dist/cli.d.ts +9 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +548 -0
- package/dist/cli.js.map +1 -0
- package/dist/config.d.ts +51 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +114 -0
- package/dist/config.js.map +1 -0
- package/dist/credentials.d.ts +52 -0
- package/dist/credentials.d.ts.map +1 -0
- package/dist/credentials.js +371 -0
- package/dist/credentials.js.map +1 -0
- package/dist/demo.d.ts +26 -0
- package/dist/demo.d.ts.map +1 -0
- package/dist/demo.js +95 -0
- package/dist/demo.js.map +1 -0
- package/dist/engine.d.ts +91 -0
- package/dist/engine.d.ts.map +1 -0
- package/dist/engine.js +1036 -0
- package/dist/engine.js.map +1 -0
- package/dist/index.d.ts +30 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +39 -0
- package/dist/index.js.map +1 -0
- package/dist/investigation-tools/aph-holdings.d.ts +61 -0
- package/dist/investigation-tools/aph-holdings.d.ts.map +1 -0
- package/dist/investigation-tools/aph-holdings.js +459 -0
- package/dist/investigation-tools/aph-holdings.js.map +1 -0
- package/dist/investigation-tools/asic-officer-lookup.d.ts +42 -0
- package/dist/investigation-tools/asic-officer-lookup.d.ts.map +1 -0
- package/dist/investigation-tools/asic-officer-lookup.js +197 -0
- package/dist/investigation-tools/asic-officer-lookup.js.map +1 -0
- package/dist/investigation-tools/asx-calendar-fetcher.d.ts +42 -0
- package/dist/investigation-tools/asx-calendar-fetcher.d.ts.map +1 -0
- package/dist/investigation-tools/asx-calendar-fetcher.js +271 -0
- package/dist/investigation-tools/asx-calendar-fetcher.js.map +1 -0
- package/dist/investigation-tools/asx-parser.d.ts +66 -0
- package/dist/investigation-tools/asx-parser.d.ts.map +1 -0
- package/dist/investigation-tools/asx-parser.js +314 -0
- package/dist/investigation-tools/asx-parser.js.map +1 -0
- package/dist/investigation-tools/bulk-asx-announcements.d.ts +53 -0
- package/dist/investigation-tools/bulk-asx-announcements.d.ts.map +1 -0
- package/dist/investigation-tools/bulk-asx-announcements.js +204 -0
- package/dist/investigation-tools/bulk-asx-announcements.js.map +1 -0
- package/dist/investigation-tools/entity-resolver.d.ts +77 -0
- package/dist/investigation-tools/entity-resolver.d.ts.map +1 -0
- package/dist/investigation-tools/entity-resolver.js +346 -0
- package/dist/investigation-tools/entity-resolver.js.map +1 -0
- package/dist/investigation-tools/hotcopper-scraper.d.ts +73 -0
- package/dist/investigation-tools/hotcopper-scraper.d.ts.map +1 -0
- package/dist/investigation-tools/hotcopper-scraper.js +318 -0
- package/dist/investigation-tools/hotcopper-scraper.js.map +1 -0
- package/dist/investigation-tools/index.d.ts +15 -0
- package/dist/investigation-tools/index.d.ts.map +1 -0
- package/dist/investigation-tools/index.js +15 -0
- package/dist/investigation-tools/index.js.map +1 -0
- package/dist/investigation-tools/insider-graph.d.ts +173 -0
- package/dist/investigation-tools/insider-graph.d.ts.map +1 -0
- package/dist/investigation-tools/insider-graph.js +732 -0
- package/dist/investigation-tools/insider-graph.js.map +1 -0
- package/dist/investigation-tools/insider-suspicion-scorer.d.ts +97 -0
- package/dist/investigation-tools/insider-suspicion-scorer.d.ts.map +1 -0
- package/dist/investigation-tools/insider-suspicion-scorer.js +327 -0
- package/dist/investigation-tools/insider-suspicion-scorer.js.map +1 -0
- package/dist/investigation-tools/multi-forum-scraper.d.ts +104 -0
- package/dist/investigation-tools/multi-forum-scraper.d.ts.map +1 -0
- package/dist/investigation-tools/multi-forum-scraper.js +415 -0
- package/dist/investigation-tools/multi-forum-scraper.js.map +1 -0
- package/dist/investigation-tools/price-fetcher.d.ts +81 -0
- package/dist/investigation-tools/price-fetcher.d.ts.map +1 -0
- package/dist/investigation-tools/price-fetcher.js +268 -0
- package/dist/investigation-tools/price-fetcher.js.map +1 -0
- package/dist/investigation-tools/shared.d.ts +39 -0
- package/dist/investigation-tools/shared.d.ts.map +1 -0
- package/dist/investigation-tools/shared.js +203 -0
- package/dist/investigation-tools/shared.js.map +1 -0
- package/dist/investigation-tools/timeline-linker.d.ts +90 -0
- package/dist/investigation-tools/timeline-linker.d.ts.map +1 -0
- package/dist/investigation-tools/timeline-linker.js +219 -0
- package/dist/investigation-tools/timeline-linker.js.map +1 -0
- package/dist/investigation-tools/volume-scanner.d.ts +70 -0
- package/dist/investigation-tools/volume-scanner.d.ts.map +1 -0
- package/dist/investigation-tools/volume-scanner.js +227 -0
- package/dist/investigation-tools/volume-scanner.js.map +1 -0
- package/dist/model.d.ts +136 -0
- package/dist/model.d.ts.map +1 -0
- package/dist/model.js +1071 -0
- package/dist/model.js.map +1 -0
- package/dist/patching.d.ts +45 -0
- package/dist/patching.d.ts.map +1 -0
- package/dist/patching.js +317 -0
- package/dist/patching.js.map +1 -0
- package/dist/prompts.d.ts +15 -0
- package/dist/prompts.d.ts.map +1 -0
- package/dist/prompts.js +351 -0
- package/dist/prompts.js.map +1 -0
- package/dist/replay-log.d.ts +54 -0
- package/dist/replay-log.d.ts.map +1 -0
- package/dist/replay-log.js +94 -0
- package/dist/replay-log.js.map +1 -0
- package/dist/runtime.d.ts +53 -0
- package/dist/runtime.d.ts.map +1 -0
- package/dist/runtime.js +259 -0
- package/dist/runtime.js.map +1 -0
- package/dist/settings.d.ts +39 -0
- package/dist/settings.d.ts.map +1 -0
- package/dist/settings.js +146 -0
- package/dist/settings.js.map +1 -0
- package/dist/tool-defs.d.ts +58 -0
- package/dist/tool-defs.d.ts.map +1 -0
- package/dist/tool-defs.js +1029 -0
- package/dist/tool-defs.js.map +1 -0
- package/dist/tools.d.ts +72 -0
- package/dist/tools.d.ts.map +1 -0
- package/dist/tools.js +1454 -0
- package/dist/tools.js.map +1 -0
- package/dist/tui.d.ts +49 -0
- package/dist/tui.d.ts.map +1 -0
- package/dist/tui.js +699 -0
- package/dist/tui.js.map +1 -0
- package/package.json +126 -0
|
@@ -0,0 +1,415 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* multi-forum-scraper.ts – Multi-Forum Scraper for ASX Ticker Sentiment & Rumor Extraction
|
|
3
|
+
*
|
|
4
|
+
* Scrapes multiple forums and social media platforms for posts related to a
|
|
5
|
+
* given ASX ticker. Supported sources:
|
|
6
|
+
*
|
|
7
|
+
* - HotCopper (https://hotcopper.com.au)
|
|
8
|
+
* - Stockhead (https://stockhead.com.au)
|
|
9
|
+
* - Twitter / X via Nitter (https://nitter.net)
|
|
10
|
+
*
|
|
11
|
+
* Extracts post metadata, performs basic keyword-based sentiment analysis, and
|
|
12
|
+
* flags rumor-type posts that may be relevant to insider-trading investigations.
|
|
13
|
+
*
|
|
14
|
+
* DISCLAIMER:
|
|
15
|
+
* This tool is provided for research and educational purposes only.
|
|
16
|
+
* Scraping the above websites may violate their respective Terms of
|
|
17
|
+
* Service. Users are solely responsible for ensuring compliance with all
|
|
18
|
+
* applicable laws, regulations, and website terms before using this tool.
|
|
19
|
+
* The authors accept no liability for misuse.
|
|
20
|
+
*/
|
|
21
|
+
import { fetchHtml, parseDate, sentimentScore, sentimentLabel, isRumor, isoNow, BULLISH_KEYWORDS, BEARISH_KEYWORDS, RUMOR_KEYWORDS, } from "./shared.js";
|
|
22
|
+
// ── Constants ───────────────────────────────────────────────────
|
|
23
|
+
const HOTCOPPER_BASE_URL = "https://hotcopper.com.au";
|
|
24
|
+
const HOTCOPPER_DISCUSSION_URL = HOTCOPPER_BASE_URL + "/asx/{ticker}/discussion/";
|
|
25
|
+
const STOCKHEAD_SEARCH_URL = "https://stockhead.com.au/search/?q={ticker}";
|
|
26
|
+
const NITTER_SEARCH_URL = "https://nitter.net/search?q=%24{ticker}+ASX";
|
|
27
|
+
/** All supported scraping sites. */
|
|
28
|
+
export const SUPPORTED_SITES = ["hotcopper", "stockhead", "twitter"];
|
|
29
|
+
const MAX_POST_TEXT_LEN = 500;
|
|
30
|
+
const DEFAULT_DELAY = 1.5;
|
|
31
|
+
// ── Helpers ─────────────────────────────────────────────────────
|
|
32
|
+
/** Truncate text to max length, breaking at the last space. */
|
|
33
|
+
function truncate(text, maxLen = MAX_POST_TEXT_LEN) {
|
|
34
|
+
if (text.length <= maxLen)
|
|
35
|
+
return text;
|
|
36
|
+
const cut = text.slice(0, maxLen);
|
|
37
|
+
const spaceIdx = cut.lastIndexOf(" ");
|
|
38
|
+
return (spaceIdx > 0 ? cut.slice(0, spaceIdx) : cut) + "…";
|
|
39
|
+
}
|
|
40
|
+
/** Round a number to 3 decimal places. */
|
|
41
|
+
function round3(n) {
|
|
42
|
+
return Math.round(n * 1000) / 1000;
|
|
43
|
+
}
|
|
44
|
+
/** Format a Date as YYYY-MM-DD. */
|
|
45
|
+
function formatDateStr(d) {
|
|
46
|
+
const y = d.getFullYear();
|
|
47
|
+
const m = String(d.getMonth() + 1).padStart(2, "0");
|
|
48
|
+
const day = String(d.getDate()).padStart(2, "0");
|
|
49
|
+
return `${y}-${m}-${day}`;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Parse HotCopper's DD/MM/YY date string into a Date.
|
|
53
|
+
* Falls back to the shared parseDate for other formats.
|
|
54
|
+
*/
|
|
55
|
+
function parseHcDate(raw) {
|
|
56
|
+
return parseDate(raw);
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Return list of matched keywords from built-in + extra keyword sets.
|
|
60
|
+
*
|
|
61
|
+
* Rumor keywords are matched by substring, sentiment keywords by word
|
|
62
|
+
* boundary, and extra user-supplied keywords by substring.
|
|
63
|
+
*/
|
|
64
|
+
function matchKeywords(text, extraKeywords) {
|
|
65
|
+
const textLower = text.toLowerCase();
|
|
66
|
+
const matched = new Set();
|
|
67
|
+
// Check built-in rumor keywords (substring match)
|
|
68
|
+
for (const kw of RUMOR_KEYWORDS) {
|
|
69
|
+
if (textLower.includes(kw.toLowerCase())) {
|
|
70
|
+
matched.add(kw);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
// Check sentiment keywords (word-level match)
|
|
74
|
+
const words = new Set(textLower.match(/[a-z]+/g) ?? []);
|
|
75
|
+
const allSentimentKw = [...BULLISH_KEYWORDS, ...BEARISH_KEYWORDS].sort();
|
|
76
|
+
for (const kw of allSentimentKw) {
|
|
77
|
+
if (words.has(kw)) {
|
|
78
|
+
matched.add(kw);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
// Check extra user-supplied keywords (substring match)
|
|
82
|
+
for (const kw of extraKeywords) {
|
|
83
|
+
if (textLower.includes(kw.toLowerCase())) {
|
|
84
|
+
matched.add(kw);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return [...matched].sort();
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Build a normalised post record.
|
|
91
|
+
*/
|
|
92
|
+
function makePost(source, ticker, date, author, title, textSnippet, url, extraKeywords) {
|
|
93
|
+
const combined = `${title} ${textSnippet}`;
|
|
94
|
+
const score = sentimentScore(combined);
|
|
95
|
+
return {
|
|
96
|
+
source,
|
|
97
|
+
ticker: ticker.toUpperCase(),
|
|
98
|
+
date,
|
|
99
|
+
author,
|
|
100
|
+
title,
|
|
101
|
+
text_snippet: truncate(textSnippet),
|
|
102
|
+
sentiment: sentimentLabel(score),
|
|
103
|
+
sentiment_score: round3(score),
|
|
104
|
+
rumor_flag: isRumor(combined),
|
|
105
|
+
keywords_matched: matchKeywords(combined, extraKeywords),
|
|
106
|
+
url,
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Fetch a page, returning a CheerioAPI or null on failure.
|
|
111
|
+
* Handles errors gracefully without throwing.
|
|
112
|
+
*/
|
|
113
|
+
async function fetchPage(url, delay, userAgent) {
|
|
114
|
+
try {
|
|
115
|
+
return await fetchHtml(url, { delay, userAgent });
|
|
116
|
+
}
|
|
117
|
+
catch {
|
|
118
|
+
return null;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
// ── Site Scrapers ───────────────────────────────────────────────
|
|
122
|
+
/**
|
|
123
|
+
* Scrape HotCopper discussion pages for the given ticker.
|
|
124
|
+
*
|
|
125
|
+
* This is a single-page scraper used by the multi-forum aggregator.
|
|
126
|
+
* For a full multi-page HotCopper scraper with thread body fetching,
|
|
127
|
+
* see {@link ../hotcopper-scraper.js}.
|
|
128
|
+
*/
|
|
129
|
+
export async function scrapeHotcopper(ticker, days, delay, extraKeywords, userAgent) {
|
|
130
|
+
const tickerUpper = ticker.toUpperCase();
|
|
131
|
+
const now = new Date();
|
|
132
|
+
const cutoff = new Date(now.getTime() - days * 24 * 60 * 60 * 1000);
|
|
133
|
+
const posts = [];
|
|
134
|
+
const url = HOTCOPPER_DISCUSSION_URL.replace("{ticker}", ticker.toLowerCase());
|
|
135
|
+
const $ = await fetchPage(url, delay, userAgent);
|
|
136
|
+
if ($ === null)
|
|
137
|
+
return posts;
|
|
138
|
+
const rows = $("table tr");
|
|
139
|
+
rows.each((_i, row) => {
|
|
140
|
+
const cells = $(row).find("td");
|
|
141
|
+
if (cells.length < 6)
|
|
142
|
+
return;
|
|
143
|
+
// Date — typically last cell
|
|
144
|
+
const dateText = $(cells[cells.length - 1]).text().trim();
|
|
145
|
+
const postDate = parseHcDate(dateText);
|
|
146
|
+
if (postDate === null)
|
|
147
|
+
return;
|
|
148
|
+
if (postDate.getTime() < cutoff.getTime())
|
|
149
|
+
return;
|
|
150
|
+
// Subject link
|
|
151
|
+
let subjectHref = "";
|
|
152
|
+
let subjectText = "";
|
|
153
|
+
let found = false;
|
|
154
|
+
cells.each((_j, cell) => {
|
|
155
|
+
if (found)
|
|
156
|
+
return;
|
|
157
|
+
const link = $(cell).find('a[href*="/threads/"]').first();
|
|
158
|
+
if (link.length > 0) {
|
|
159
|
+
subjectHref = link.attr("href") ?? "";
|
|
160
|
+
subjectText = link.text().trim();
|
|
161
|
+
found = true;
|
|
162
|
+
}
|
|
163
|
+
});
|
|
164
|
+
if (!found)
|
|
165
|
+
return;
|
|
166
|
+
const threadTitle = subjectText.replace(/^Re:\s*/i, "");
|
|
167
|
+
let threadUrl = subjectHref;
|
|
168
|
+
if (threadUrl.startsWith("/")) {
|
|
169
|
+
threadUrl = HOTCOPPER_BASE_URL + threadUrl;
|
|
170
|
+
}
|
|
171
|
+
// Poster
|
|
172
|
+
let poster = "";
|
|
173
|
+
const posterLink = $(row).find('a[href*="users="]').first();
|
|
174
|
+
if (posterLink.length > 0) {
|
|
175
|
+
poster = posterLink.text().trim();
|
|
176
|
+
}
|
|
177
|
+
posts.push(makePost("hotcopper", tickerUpper, formatDateStr(postDate), poster, threadTitle, threadTitle, threadUrl, extraKeywords));
|
|
178
|
+
});
|
|
179
|
+
return posts;
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Scrape Stockhead search results for the given ticker.
|
|
183
|
+
*/
|
|
184
|
+
export async function scrapeStockhead(ticker, days, delay, extraKeywords, userAgent) {
|
|
185
|
+
const tickerUpper = ticker.toUpperCase();
|
|
186
|
+
const now = new Date();
|
|
187
|
+
const cutoff = new Date(now.getTime() - days * 24 * 60 * 60 * 1000);
|
|
188
|
+
const posts = [];
|
|
189
|
+
const url = STOCKHEAD_SEARCH_URL.replace("{ticker}", tickerUpper);
|
|
190
|
+
const $ = await fetchPage(url, delay, userAgent);
|
|
191
|
+
if ($ === null)
|
|
192
|
+
return posts;
|
|
193
|
+
// Stockhead search results are typically article cards
|
|
194
|
+
let articles = $("article, div.search-result, div.post-item, li.result-item");
|
|
195
|
+
if (articles.length === 0) {
|
|
196
|
+
// Fallback: try generic heading + link patterns
|
|
197
|
+
articles = $("h2, h3, h4");
|
|
198
|
+
}
|
|
199
|
+
articles.each((_i, article) => {
|
|
200
|
+
// Extract title
|
|
201
|
+
const titleEl = $(article).find("h2, h3, h4, a").first();
|
|
202
|
+
if (titleEl.length === 0)
|
|
203
|
+
return;
|
|
204
|
+
const title = titleEl.text().trim();
|
|
205
|
+
if (!title)
|
|
206
|
+
return;
|
|
207
|
+
// Extract link
|
|
208
|
+
let articleUrl = "";
|
|
209
|
+
const linkEl = $(article).is("a")
|
|
210
|
+
? $(article)
|
|
211
|
+
: $(article).find("a[href]").first();
|
|
212
|
+
if (linkEl.length === 0) {
|
|
213
|
+
const parentLink = titleEl.closest("a");
|
|
214
|
+
if (parentLink.length > 0) {
|
|
215
|
+
articleUrl = parentLink.attr("href") ?? "";
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
else {
|
|
219
|
+
articleUrl = linkEl.attr("href") ?? "";
|
|
220
|
+
}
|
|
221
|
+
if (articleUrl.startsWith("/")) {
|
|
222
|
+
articleUrl = "https://stockhead.com.au" + articleUrl;
|
|
223
|
+
}
|
|
224
|
+
// Extract snippet
|
|
225
|
+
let snippet = title;
|
|
226
|
+
const excerptEl = $(article).find('p, span[class*="excerpt"], div[class*="excerpt"]').first();
|
|
227
|
+
if (excerptEl.length > 0) {
|
|
228
|
+
const excerptText = excerptEl.text().trim();
|
|
229
|
+
if (excerptText)
|
|
230
|
+
snippet = excerptText;
|
|
231
|
+
}
|
|
232
|
+
else {
|
|
233
|
+
const pEl = $(article).find("p").first();
|
|
234
|
+
if (pEl.length > 0) {
|
|
235
|
+
const pText = pEl.text().trim();
|
|
236
|
+
if (pText)
|
|
237
|
+
snippet = pText;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
// Extract date
|
|
241
|
+
let dateStr = "";
|
|
242
|
+
const timeEl = $(article).find("time").first();
|
|
243
|
+
if (timeEl.length > 0 && timeEl.attr("datetime")) {
|
|
244
|
+
try {
|
|
245
|
+
const dt = new Date(timeEl.attr("datetime").replace("Z", "+00:00"));
|
|
246
|
+
if (!isNaN(dt.getTime())) {
|
|
247
|
+
if (dt.getTime() < cutoff.getTime())
|
|
248
|
+
return;
|
|
249
|
+
dateStr = formatDateStr(dt);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
catch {
|
|
253
|
+
dateStr = formatDateStr(now);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
if (!dateStr) {
|
|
257
|
+
dateStr = formatDateStr(now);
|
|
258
|
+
}
|
|
259
|
+
// Author
|
|
260
|
+
let author = "Stockhead";
|
|
261
|
+
const authorEl = $(article).find('[class*="author"]').first();
|
|
262
|
+
if (authorEl.length > 0) {
|
|
263
|
+
const authorText = authorEl.text().trim();
|
|
264
|
+
if (authorText)
|
|
265
|
+
author = authorText;
|
|
266
|
+
}
|
|
267
|
+
posts.push(makePost("stockhead", tickerUpper, dateStr, author, title, snippet, articleUrl, extraKeywords));
|
|
268
|
+
});
|
|
269
|
+
return posts;
|
|
270
|
+
}
|
|
271
|
+
/**
|
|
272
|
+
* Scrape Twitter/X posts via Nitter for the given ticker.
|
|
273
|
+
*/
|
|
274
|
+
export async function scrapeTwitter(ticker, days, delay, extraKeywords, userAgent) {
|
|
275
|
+
const tickerUpper = ticker.toUpperCase();
|
|
276
|
+
const now = new Date();
|
|
277
|
+
const cutoff = new Date(now.getTime() - days * 24 * 60 * 60 * 1000);
|
|
278
|
+
const posts = [];
|
|
279
|
+
const url = NITTER_SEARCH_URL.replace("{ticker}", tickerUpper);
|
|
280
|
+
const $ = await fetchPage(url, delay, userAgent);
|
|
281
|
+
if ($ === null)
|
|
282
|
+
return posts;
|
|
283
|
+
// Nitter tweet containers
|
|
284
|
+
let tweetEls = $("div.timeline-item, div.tweet-card, div.tweet");
|
|
285
|
+
if (tweetEls.length === 0) {
|
|
286
|
+
// Alternative selectors
|
|
287
|
+
tweetEls = $("article, div.post");
|
|
288
|
+
}
|
|
289
|
+
tweetEls.each((_i, tweetEl) => {
|
|
290
|
+
// Username
|
|
291
|
+
const userEl = $(tweetEl).find('[class*="username"]').first();
|
|
292
|
+
const username = userEl.length > 0 ? userEl.text().trim() : "unknown";
|
|
293
|
+
// Tweet text
|
|
294
|
+
let textEl = $(tweetEl).find('[class*="content"]').first();
|
|
295
|
+
if (textEl.length === 0) {
|
|
296
|
+
textEl = $(tweetEl).find("p").first();
|
|
297
|
+
}
|
|
298
|
+
if (textEl.length === 0)
|
|
299
|
+
return;
|
|
300
|
+
const text = textEl.text().trim();
|
|
301
|
+
if (!text)
|
|
302
|
+
return;
|
|
303
|
+
// Date
|
|
304
|
+
let dateStr = "";
|
|
305
|
+
const timeEl = $(tweetEl).find("time").first();
|
|
306
|
+
if (timeEl.length > 0 && timeEl.attr("datetime")) {
|
|
307
|
+
try {
|
|
308
|
+
const dt = new Date(timeEl.attr("datetime").replace("Z", "+00:00"));
|
|
309
|
+
if (!isNaN(dt.getTime())) {
|
|
310
|
+
if (dt.getTime() < cutoff.getTime())
|
|
311
|
+
return;
|
|
312
|
+
dateStr = formatDateStr(dt);
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
catch {
|
|
316
|
+
dateStr = formatDateStr(now);
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
if (!dateStr) {
|
|
320
|
+
dateStr = formatDateStr(now);
|
|
321
|
+
}
|
|
322
|
+
// Tweet URL
|
|
323
|
+
let tweetUrl = "";
|
|
324
|
+
const tweetLink = $(tweetEl).find('a[href*="/status/"]').first();
|
|
325
|
+
if (tweetLink.length > 0) {
|
|
326
|
+
const href = tweetLink.attr("href") ?? "";
|
|
327
|
+
if (href.startsWith("/")) {
|
|
328
|
+
tweetUrl = "https://nitter.net" + href;
|
|
329
|
+
}
|
|
330
|
+
else {
|
|
331
|
+
tweetUrl = href;
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
posts.push(makePost("twitter", tickerUpper, dateStr, username, "", text, tweetUrl, extraKeywords));
|
|
335
|
+
});
|
|
336
|
+
return posts;
|
|
337
|
+
}
|
|
338
|
+
// ── Aggregation ─────────────────────────────────────────────────
|
|
339
|
+
/**
|
|
340
|
+
* Build an aggregate summary across all scraped posts.
|
|
341
|
+
*/
|
|
342
|
+
export function aggregateSummary(posts) {
|
|
343
|
+
// Posts per source
|
|
344
|
+
const sourceCounts = {};
|
|
345
|
+
for (const p of posts) {
|
|
346
|
+
sourceCounts[p.source] = (sourceCounts[p.source] ?? 0) + 1;
|
|
347
|
+
}
|
|
348
|
+
// Sentiment distribution
|
|
349
|
+
const sentimentDist = { bullish: 0, bearish: 0, neutral: 0 };
|
|
350
|
+
for (const p of posts) {
|
|
351
|
+
const label = p.sentiment;
|
|
352
|
+
if (label in sentimentDist) {
|
|
353
|
+
sentimentDist[label] += 1;
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
// Rumor count
|
|
357
|
+
const rumorCount = posts.filter((p) => p.rumor_flag).length;
|
|
358
|
+
// Overall average sentiment score
|
|
359
|
+
const scores = posts.map((p) => p.sentiment_score);
|
|
360
|
+
const avgScore = scores.length > 0
|
|
361
|
+
? round3(scores.reduce((a, b) => a + b, 0) / scores.length)
|
|
362
|
+
: 0.0;
|
|
363
|
+
return {
|
|
364
|
+
total_posts: posts.length,
|
|
365
|
+
posts_per_source: sourceCounts,
|
|
366
|
+
sentiment_distribution: sentimentDist,
|
|
367
|
+
average_sentiment_score: avgScore,
|
|
368
|
+
rumor_count: rumorCount,
|
|
369
|
+
};
|
|
370
|
+
}
|
|
371
|
+
// ── Main Scraper ────────────────────────────────────────────────
|
|
372
|
+
/** Map of site names to their scraper functions. */
|
|
373
|
+
const SCRAPERS = {
|
|
374
|
+
hotcopper: scrapeHotcopper,
|
|
375
|
+
stockhead: scrapeStockhead,
|
|
376
|
+
twitter: scrapeTwitter,
|
|
377
|
+
};
|
|
378
|
+
/**
|
|
379
|
+
* Scrape multiple forums and social media for ASX ticker sentiment and rumor data.
|
|
380
|
+
*
|
|
381
|
+
* @param opts - Scraper options.
|
|
382
|
+
* @returns Structured result with posts from all requested sites and aggregate summary.
|
|
383
|
+
*/
|
|
384
|
+
export async function scrapeMultiForum(opts) {
|
|
385
|
+
const ticker = opts.ticker.toUpperCase().trim();
|
|
386
|
+
const sites = opts.sites ?? SUPPORTED_SITES;
|
|
387
|
+
const days = opts.days ?? 7;
|
|
388
|
+
const extraKeywords = opts.extraKeywords ?? [];
|
|
389
|
+
const delay = opts.delay ?? DEFAULT_DELAY;
|
|
390
|
+
const userAgent = opts.userAgent;
|
|
391
|
+
const allPosts = [];
|
|
392
|
+
for (const site of sites) {
|
|
393
|
+
const scraperFn = SCRAPERS[site];
|
|
394
|
+
if (!scraperFn)
|
|
395
|
+
continue;
|
|
396
|
+
try {
|
|
397
|
+
const sitePosts = await scraperFn(ticker, days, delay, extraKeywords, userAgent);
|
|
398
|
+
allPosts.push(...sitePosts);
|
|
399
|
+
}
|
|
400
|
+
catch {
|
|
401
|
+
// Continue with other sites on failure
|
|
402
|
+
continue;
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
const summary = aggregateSummary(allPosts);
|
|
406
|
+
return {
|
|
407
|
+
ticker,
|
|
408
|
+
scraped_at: isoNow(),
|
|
409
|
+
period: `${days} days`,
|
|
410
|
+
sites_requested: sites,
|
|
411
|
+
posts: allPosts,
|
|
412
|
+
summary,
|
|
413
|
+
};
|
|
414
|
+
}
|
|
415
|
+
//# sourceMappingURL=multi-forum-scraper.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"multi-forum-scraper.js","sourceRoot":"","sources":["../../src/investigation-tools/multi-forum-scraper.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAGH,OAAO,EACL,SAAS,EACT,SAAS,EACT,cAAc,EACd,cAAc,EACd,OAAO,EACP,MAAM,EACN,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,GAEf,MAAM,aAAa,CAAC;AAErB,mEAAmE;AAEnE,MAAM,kBAAkB,GAAG,0BAA0B,CAAC;AACtD,MAAM,wBAAwB,GAAG,kBAAkB,GAAG,2BAA2B,CAAC;AAElF,MAAM,oBAAoB,GAAG,6CAA6C,CAAC;AAE3E,MAAM,iBAAiB,GAAG,6CAA6C,CAAC;AAExE,oCAAoC;AACpC,MAAM,CAAC,MAAM,eAAe,GAAsB,CAAC,WAAW,EAAE,WAAW,EAAE,SAAS,CAAU,CAAC;AAEjG,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAC9B,MAAM,aAAa,GAAG,GAAG,CAAC;AAgE1B,mEAAmE;AAEnE,+DAA+D;AAC/D,SAAS,QAAQ,CAAC,IAAY,EAAE,SAAiB,iBAAiB;IAChE,IAAI,IAAI,CAAC,MAAM,IAAI,MAAM;QAAE,OAAO,IAAI,CAAC;IACvC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACtC,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;AAC7D,CAAC;AAED,0CAA0C;AAC1C,SAAS,MAAM,CAAC,CAAS;IACvB,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;AACrC,CAAC;AAED,mCAAmC;AACnC,SAAS,aAAa,CAAC,CAAO;IAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;IAC1B,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACpD,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACjD,OAAO,GAAG,CAAC,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC;AAC5B,CAAC;AAED;;;GAGG;AACH,SAAS,WAAW,CAAC,GAAW;IAC9B,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC;AACxB,CAAC;AAED;;;;;GAKG;AACH,SAAS,aAAa,CAAC,IAAY,EAAE,aAAuB;IAC1D,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IACrC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAElC,kDAAkD;IAClD,KAAK,MAAM,EAAE,IAAI,cAAc,EAAE,CAAC;QAChC,IAAI,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YACzC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAED,8CAA8C;IAC9C,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;IACxD,MAAM,cAAc,GAAG,CAAC,GAAG,gBAAgB,EAAE,GAAG,gBAAgB,CAAC,CAAC,IAAI,EAAE,CAAC;IACzE,KAAK,MAAM,EAAE,IAAI,cAAc,EAAE,CAAC;QAChC,IAAI,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;YAClB,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAED,uDAAuD;IACvD,KAAK,MAAM,EAAE,IAAI,aAAa,EAAE,CAAC;QAC/B,IAAI,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YACzC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;AAC7B,CAAC;AAED;;GAEG;AACH,SAAS,QAAQ,CACf,MAAc,EACd,MAAc,EACd,IAAY,EACZ,MAAc,EACd,KAAa,EACb,WAAmB,EACnB,GAAW,EACX,aAAuB;IAEvB,MAAM,QAAQ,GAAG,GAAG,KAAK,IAAI,WAAW,EAAE,CAAC;IAC3C,MAAM,KAAK,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACvC,OAAO;QACL,MAAM;QACN,MAAM,EAAE,MAAM,CAAC,WAAW,EAAE;QAC5B,IAAI;QACJ,MAAM;QACN,KAAK;QACL,YAAY,EAAE,QAAQ,CAAC,WAAW,CAAC;QACnC,SAAS,EAAE,cAAc,CAAC,KAAK,CAAC;QAChC,eAAe,EAAE,MAAM,CAAC,KAAK,CAAC;QAC9B,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC;QAC7B,gBAAgB,EAAE,aAAa,CAAC,QAAQ,EAAE,aAAa,CAAC;QACxD,GAAG;KACJ,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,SAAS,CACtB,GAAW,EACX,KAAa,EACb,SAAkB;IAElB,IAAI,CAAC;QACH,OAAO,MAAM,SAAS,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;IACpD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,mEAAmE;AAEnE;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAc,EACd,IAAY,EACZ,KAAa,EACb,aAAuB,EACvB,SAAkB;IAElB,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;IACzC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IACvB,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;IACpE,MAAM,KAAK,GAAgB,EAAE,CAAC;IAE9B,MAAM,GAAG,GAAG,wBAAwB,CAAC,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;IAE/E,MAAM,CAAC,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;IACjD,IAAI,CAAC,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAE7B,MAAM,IAAI,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;IAC3B,IAAI,CAAC,IAAI,CAAC,CAAC,EAAU,EAAE,GAAY,EAAE,EAAE;QACrC,MAAM,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO;QAE7B,6BAA6B;QAC7B,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAC1D,MAAM,QAAQ,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;QACvC,IAAI,QAAQ,KAAK,IAAI;YAAE,OAAO;QAC9B,IAAI,QAAQ,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,OAAO,EAAE;YAAE,OAAO;QAElD,eAAe;QACf,IAAI,WAAW,GAAG,EAAE,CAAC;QACrB,IAAI,WAAW,GAAG,EAAE,CAAC;QACrB,IAAI,KAAK,GAAG,KAAK,CAAC;QAElB,KAAK,CAAC,IAAI,CAAC,CAAC,EAAU,EAAE,IAAa,EAAE,EAAE;YACvC,IAAI,KAAK;gBAAE,OAAO;YAClB,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC,KAAK,EAAE,CAAC;YAC1D,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpB,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;gBACtC,WAAW,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBACjC,KAAK,GAAG,IAAI,CAAC;YACf,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,KAAK;YAAE,OAAO;QAEnB,MAAM,WAAW,GAAG,WAAW,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QACxD,IAAI,SAAS,GAAG,WAAW,CAAC;QAC5B,IAAI,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC9B,SAAS,GAAG,kBAAkB,GAAG,SAAS,CAAC;QAC7C,CAAC;QAED,SAAS;QACT,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,MAAM,UAAU,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,KAAK,EAAE,CAAC;QAC5D,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QACpC,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,QAAQ,CACjB,WAAW,EACX,WAAW,EACX,aAAa,CAAC,QAAQ,CAAC,EACvB,MAAM,EACN,WAAW,EACX,WAAW,EACX,SAAS,EACT,aAAa,CACd,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAc,EACd,IAAY,EACZ,KAAa,EACb,aAAuB,EACvB,SAAkB;IAElB,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;IACzC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IACvB,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;IACpE,MAAM,KAAK,GAAgB,EAAE,CAAC;IAE9B,MAAM,GAAG,GAAG,oBAAoB,CAAC,OAAO,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;IAElE,MAAM,CAAC,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;IACjD,IAAI,CAAC,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAE7B,uDAAuD;IACvD,IAAI,QAAQ,GAAG,CAAC,CAAC,2DAA2D,CAAC,CAAC;IAC9E,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,gDAAgD;QAChD,QAAQ,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC;IAC7B,CAAC;IAED,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAU,EAAE,OAAgB,EAAE,EAAE;QAC7C,gBAAgB;QAChB,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,KAAK,EAAE,CAAC;QACzD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACjC,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QACpC,IAAI,CAAC,KAAK;YAAE,OAAO;QAEnB,eAAe;QACf,IAAI,UAAU,GAAG,EAAE,CAAC;QACpB,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC;YAC/B,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;YACZ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC;QACvC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YACxC,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YAC7C,CAAC;QACH,CAAC;aAAM,CAAC;YACN,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACzC,CAAC;QACD,IAAI,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/B,UAAU,GAAG,0BAA0B,GAAG,UAAU,CAAC;QACvD,CAAC;QAED,kBAAkB;QAClB,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAC/B,kDAAkD,CACnD,CAAC,KAAK,EAAE,CAAC;QACV,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,WAAW,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAC5C,IAAI,WAAW;gBAAE,OAAO,GAAG,WAAW,CAAC;QACzC,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC;YACzC,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACnB,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBAChC,IAAI,KAAK;oBAAE,OAAO,GAAG,KAAK,CAAC;YAC7B,CAAC;QACH,CAAC;QAED,eAAe;QACf,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YACjD,IAAI,CAAC;gBACH,MAAM,EAAE,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAE,CAAC,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC;gBACrE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;oBACzB,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,OAAO,EAAE;wBAAE,OAAO;oBAC5C,OAAO,GAAG,aAAa,CAAC,EAAE,CAAC,CAAC;gBAC9B,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QACD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;QAC/B,CAAC;QAED,SAAS;QACT,IAAI,MAAM,GAAG,WAAW,CAAC;QACzB,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,KAAK,EAAE,CAAC;QAC9D,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAC1C,IAAI,UAAU;gBAAE,MAAM,GAAG,UAAU,CAAC;QACtC,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,QAAQ,CACjB,WAAW,EACX,WAAW,EACX,OAAO,EACP,MAAM,EACN,KAAK,EACL,OAAO,EACP,UAAU,EACV,aAAa,CACd,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAc,EACd,IAAY,EACZ,KAAa,EACb,aAAuB,EACvB,SAAkB;IAElB,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;IACzC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IACvB,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;IACpE,MAAM,KAAK,GAAgB,EAAE,CAAC;IAE9B,MAAM,GAAG,GAAG,iBAAiB,CAAC,OAAO,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;IAE/D,MAAM,CAAC,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;IACjD,IAAI,CAAC,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAE7B,0BAA0B;IAC1B,IAAI,QAAQ,GAAG,CAAC,CAAC,8CAA8C,CAAC,CAAC;IACjE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,wBAAwB;QACxB,QAAQ,GAAG,CAAC,CAAC,mBAAmB,CAAC,CAAC;IACpC,CAAC;IAED,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAU,EAAE,OAAgB,EAAE,EAAE;QAC7C,WAAW;QACX,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,KAAK,EAAE,CAAC;QAC9D,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAEtE,aAAa;QACb,IAAI,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,KAAK,EAAE,CAAC;QAC3D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC;QACxC,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAChC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAClC,IAAI,CAAC,IAAI;YAAE,OAAO;QAElB,OAAO;QACP,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC;QAC/C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YACjD,IAAI,CAAC;gBACH,MAAM,EAAE,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAE,CAAC,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC;gBACrE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;oBACzB,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,OAAO,EAAE;wBAAE,OAAO;oBAC5C,OAAO,GAAG,aAAa,CAAC,EAAE,CAAC,CAAC;gBAC9B,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QACD,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;QAC/B,CAAC;QAED,YAAY;QACZ,IAAI,QAAQ,GAAG,EAAE,CAAC;QAClB,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,KAAK,EAAE,CAAC;QACjE,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YAC1C,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACzB,QAAQ,GAAG,oBAAoB,GAAG,IAAI,CAAC;YACzC,CAAC;iBAAM,CAAC;gBACN,QAAQ,GAAG,IAAI,CAAC;YAClB,CAAC;QACH,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,QAAQ,CACjB,SAAS,EACT,WAAW,EACX,OAAO,EACP,QAAQ,EACR,EAAE,EACF,IAAI,EACJ,QAAQ,EACR,aAAa,CACd,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC;AACf,CAAC;AAED,mEAAmE;AAEnE;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAkB;IACjD,mBAAmB;IACnB,MAAM,YAAY,GAA2B,EAAE,CAAC;IAChD,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAC7D,CAAC;IAED,yBAAyB;IACzB,MAAM,aAAa,GAA0B,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IACpF,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,CAAC,CAAC,SAAwC,CAAC;QACzD,IAAI,KAAK,IAAI,aAAa,EAAE,CAAC;YAC3B,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,cAAc;IACd,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC;IAE5D,kCAAkC;IAClC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC;QAChC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QAC3D,CAAC,CAAC,GAAG,CAAC;IAER,OAAO;QACL,WAAW,EAAE,KAAK,CAAC,MAAM;QACzB,gBAAgB,EAAE,YAAY;QAC9B,sBAAsB,EAAE,aAAa;QACrC,uBAAuB,EAAE,QAAQ;QACjC,WAAW,EAAE,UAAU;KACxB,CAAC;AACJ,CAAC;AAED,mEAAmE;AAEnE,oDAAoD;AACpD,MAAM,QAAQ,GASV;IACF,SAAS,EAAE,eAAe;IAC1B,SAAS,EAAE,eAAe;IAC1B,OAAO,EAAE,aAAa;CACvB,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAuB;IAEvB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;IAChD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAK,eAAyC,CAAC;IACvE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;IAC5B,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,EAAE,CAAC;IAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,aAAa,CAAC;IAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;IAEjC,MAAM,QAAQ,GAAgB,EAAE,CAAC;IAEjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,CAAC,SAAS;YAAE,SAAS;QAEzB,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,aAAa,EAAE,SAAS,CAAC,CAAC;YACjF,QAAQ,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;QAC9B,CAAC;QAAC,MAAM,CAAC;YACP,uCAAuC;YACvC,SAAS;QACX,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAE3C,OAAO;QACL,MAAM;QACN,UAAU,EAAE,MAAM,EAAE;QACpB,MAAM,EAAE,GAAG,IAAI,OAAO;QACtB,eAAe,EAAE,KAAK;QACtB,KAAK,EAAE,QAAQ;QACf,OAAO;KACR,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* price-fetcher.ts – ASX OHLCV price fetcher with anomaly detection.
|
|
3
|
+
*
|
|
4
|
+
* Fetches historical OHLCV data from Yahoo Finance for Australian (ASX)
|
|
5
|
+
* tickers, computes derived metrics, and flags trading anomalies.
|
|
6
|
+
*/
|
|
7
|
+
/** Raw OHLCV bar from Yahoo Finance. */
|
|
8
|
+
export interface OhlcvBar {
|
|
9
|
+
date: string;
|
|
10
|
+
open: number;
|
|
11
|
+
high: number;
|
|
12
|
+
low: number;
|
|
13
|
+
close: number;
|
|
14
|
+
volume: number;
|
|
15
|
+
}
|
|
16
|
+
/** OHLCV bar with computed metrics and anomaly flags. */
|
|
17
|
+
export interface PriceMetrics {
|
|
18
|
+
ticker: string;
|
|
19
|
+
date: string;
|
|
20
|
+
open: number;
|
|
21
|
+
high: number;
|
|
22
|
+
low: number;
|
|
23
|
+
close: number;
|
|
24
|
+
volume: number;
|
|
25
|
+
dailyReturnPct: number | null;
|
|
26
|
+
volumeRatio: number | null;
|
|
27
|
+
volatility: number | null;
|
|
28
|
+
gapPct: number | null;
|
|
29
|
+
anomalyFlags: string[];
|
|
30
|
+
}
|
|
31
|
+
/** Per-ticker summary of the requested period. */
|
|
32
|
+
export interface TickerSummary {
|
|
33
|
+
ticker: string;
|
|
34
|
+
avgVolume: number;
|
|
35
|
+
avgReturn: number;
|
|
36
|
+
maxReturn: number;
|
|
37
|
+
minReturn: number;
|
|
38
|
+
anomalyCount: number;
|
|
39
|
+
periodReturn: number;
|
|
40
|
+
}
|
|
41
|
+
export declare const ANOMALY_THRESHOLDS: {
|
|
42
|
+
readonly volume_spike: 2;
|
|
43
|
+
readonly price_surge: 5;
|
|
44
|
+
readonly price_drop: -5;
|
|
45
|
+
readonly gap_up: 3;
|
|
46
|
+
readonly gap_down: -3;
|
|
47
|
+
readonly volatility_spike_factor: 2;
|
|
48
|
+
};
|
|
49
|
+
export declare const ROLLING_VOLUME_WINDOW = 20;
|
|
50
|
+
export declare const ROLLING_VOLATILITY_WINDOW = 20;
|
|
51
|
+
export declare const VOLATILITY_BASELINE_WINDOW = 60;
|
|
52
|
+
export declare const VALID_PERIODS: Set<string>;
|
|
53
|
+
export declare const VALID_INTERVALS: Set<string>;
|
|
54
|
+
/**
|
|
55
|
+
* Fetch OHLCV data from Yahoo Finance for the given ASX tickers.
|
|
56
|
+
*
|
|
57
|
+
* Each ticker is automatically suffixed with `.AX` if needed.
|
|
58
|
+
* Returns a mapping of clean ticker → OhlcvBar[].
|
|
59
|
+
*/
|
|
60
|
+
export declare function fetchOhlcv(tickers: string[], period?: string, interval?: string): Promise<Map<string, OhlcvBar[]>>;
|
|
61
|
+
/**
|
|
62
|
+
* Compute derived metrics and anomaly flags for an array of OHLCV bars.
|
|
63
|
+
*
|
|
64
|
+
* Metrics:
|
|
65
|
+
* - `dailyReturnPct`: percentage change in close vs previous close
|
|
66
|
+
* - `volumeRatio`: current volume / 20-day rolling mean volume
|
|
67
|
+
* - `volatility`: 20-day rolling sample std of daily returns
|
|
68
|
+
* - `gapPct`: percentage gap between open and previous close
|
|
69
|
+
* - `anomalyFlags`: list of triggered threshold names
|
|
70
|
+
*/
|
|
71
|
+
export declare function computeMetrics(ticker: string, bars: OhlcvBar[]): PriceMetrics[];
|
|
72
|
+
/**
|
|
73
|
+
* Merge per-ticker data into a flat array of PriceMetrics.
|
|
74
|
+
* Optionally filter to only rows with at least one anomaly flag.
|
|
75
|
+
*/
|
|
76
|
+
export declare function buildOutput(allData: Map<string, OhlcvBar[]>, anomaliesOnly?: boolean): PriceMetrics[];
|
|
77
|
+
/**
|
|
78
|
+
* Produce a per-ticker summary from OHLCV data.
|
|
79
|
+
*/
|
|
80
|
+
export declare function buildSummary(allData: Map<string, OhlcvBar[]>): TickerSummary[];
|
|
81
|
+
//# sourceMappingURL=price-fetcher.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"price-fetcher.d.ts","sourceRoot":"","sources":["../../src/investigation-tools/price-fetcher.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAUH,wCAAwC;AACxC,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,yDAAyD;AACzD,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,kDAAkD;AAClD,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB;AAuBD,eAAO,MAAM,kBAAkB;;;;;;;CAOrB,CAAC;AAEX,eAAO,MAAM,qBAAqB,KAAK,CAAC;AACxC,eAAO,MAAM,yBAAyB,KAAK,CAAC;AAC5C,eAAO,MAAM,0BAA0B,KAAK,CAAC;AAE7C,eAAO,MAAM,aAAa,aAExB,CAAC;AAEH,eAAO,MAAM,eAAe,aAG1B,CAAC;AA+FH;;;;;GAKG;AACH,wBAAsB,UAAU,CAC9B,OAAO,EAAE,MAAM,EAAE,EACjB,MAAM,GAAE,MAAc,EACtB,QAAQ,GAAE,MAAa,GACtB,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,CAiClC;AAID;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAC5B,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,QAAQ,EAAE,GACf,YAAY,EAAE,CAsFhB;AAID;;;GAGG;AACH,wBAAgB,WAAW,CACzB,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,EAChC,aAAa,GAAE,OAAe,GAC7B,YAAY,EAAE,CAYhB;AAED;;GAEG;AACH,wBAAgB,YAAY,CAC1B,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,GAC/B,aAAa,EAAE,CA0CjB"}
|