clawmoney 0.17.44 → 0.17.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/task/daemon.js
CHANGED
|
@@ -23,6 +23,8 @@
|
|
|
23
23
|
import { readFileSync, writeFileSync, unlinkSync, existsSync, appendFileSync, mkdirSync } from "node:fs";
|
|
24
24
|
import { join } from "node:path";
|
|
25
25
|
import { homedir } from "node:os";
|
|
26
|
+
import { fileURLToPath } from "node:url";
|
|
27
|
+
import { execFileSync } from "node:child_process";
|
|
26
28
|
import YAML from "yaml";
|
|
27
29
|
const TASK_LOG_FILE = join(homedir(), ".clawmoney", "task.log");
|
|
28
30
|
function tsLine(level, msg) {
|
|
@@ -40,11 +42,11 @@ function logToFile(level, ...args) {
|
|
|
40
42
|
/* best effort */
|
|
41
43
|
}
|
|
42
44
|
}
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
console.
|
|
46
|
-
console.
|
|
47
|
-
|
|
45
|
+
function installFileLogger() {
|
|
46
|
+
console.log = (...args) => logToFile("INFO", ...args);
|
|
47
|
+
console.warn = (...args) => logToFile("WARN", ...args);
|
|
48
|
+
console.error = (...args) => logToFile("ERROR", ...args);
|
|
49
|
+
}
|
|
48
50
|
import { TaskWsClient } from "./ws-client.js";
|
|
49
51
|
import { getSkill, listSkills } from "./skills/index.js";
|
|
50
52
|
const CONFIG_DIR = join(homedir(), ".clawmoney");
|
|
@@ -185,7 +187,47 @@ async function handleTaskRequest(ws, req) {
|
|
|
185
187
|
clearTimeout(timer);
|
|
186
188
|
}
|
|
187
189
|
}
|
|
190
|
+
function applySystemProxy() {
|
|
191
|
+
// bnbot's publicScrapers honor https_proxy/http_proxy/all_proxy to reach
|
|
192
|
+
// censored public APIs (wiki/google/...). The daemon's own WS uses the
|
|
193
|
+
// `ws` lib which ignores these env vars, so this only routes child bnbot
|
|
194
|
+
// fetches through the proxy. Auto-detect the macOS system proxy when the
|
|
195
|
+
// operator hasn't set one explicitly.
|
|
196
|
+
if (process.env.https_proxy || process.env.HTTPS_PROXY ||
|
|
197
|
+
process.env.all_proxy || process.env.ALL_PROXY) {
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
200
|
+
if (process.platform !== "darwin")
|
|
201
|
+
return;
|
|
202
|
+
try {
|
|
203
|
+
const out = execFileSync("scutil", ["--proxy"], { encoding: "utf8", timeout: 3000 });
|
|
204
|
+
const get = (key) => out.match(new RegExp(`\\b${key}\\s*:\\s*(\\S+)`))?.[1];
|
|
205
|
+
let url;
|
|
206
|
+
if (get("HTTPSEnable") === "1" && get("HTTPSProxy")) {
|
|
207
|
+
url = `http://${get("HTTPSProxy")}:${get("HTTPSPort") ?? "0"}`;
|
|
208
|
+
}
|
|
209
|
+
else if (get("HTTPEnable") === "1" && get("HTTPProxy")) {
|
|
210
|
+
url = `http://${get("HTTPProxy")}:${get("HTTPPort") ?? "0"}`;
|
|
211
|
+
}
|
|
212
|
+
else if (get("SOCKSEnable") === "1" && get("SOCKSProxy")) {
|
|
213
|
+
url = `socks5://${get("SOCKSProxy")}:${get("SOCKSPort") ?? "0"}`;
|
|
214
|
+
}
|
|
215
|
+
if (url) {
|
|
216
|
+
process.env.https_proxy = url;
|
|
217
|
+
process.env.http_proxy = url;
|
|
218
|
+
process.env.all_proxy = url;
|
|
219
|
+
console.log(`[task] auto-detected system proxy ${url} (routing skill fetches through it)`);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
catch (err) {
|
|
223
|
+
console.error(`[task] system proxy detection skipped: ${err instanceof Error ? err.message : String(err)}`);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
188
226
|
function main() {
|
|
227
|
+
// Daemon was started as a script (stdio:"ignore"); from here on, all
|
|
228
|
+
// log lines should land in ~/.clawmoney/task.log.
|
|
229
|
+
installFileLogger();
|
|
230
|
+
applySystemProxy();
|
|
189
231
|
const existing = readTaskPid();
|
|
190
232
|
if (existing !== null && isPidAlive(existing)) {
|
|
191
233
|
console.error(`[task] already running (PID ${existing})`);
|
|
@@ -243,4 +285,9 @@ function main() {
|
|
|
243
285
|
process.on("SIGTERM", () => shutdown("SIGTERM"));
|
|
244
286
|
ws.start();
|
|
245
287
|
}
|
|
246
|
-
main()
|
|
288
|
+
// Only run main() when invoked as a script (`node daemon.js`), not when
|
|
289
|
+
// the module is imported (e.g. by src/commands/task.ts which only wants
|
|
290
|
+
// to use readTaskPid / isPidAlive helpers).
|
|
291
|
+
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
|
292
|
+
main();
|
|
293
|
+
}
|
|
@@ -7,6 +7,13 @@ const symbol = (i) => reqStr(i, ["symbol", "ticker"], "symbol");
|
|
|
7
7
|
function openSkill(label, base, positional = () => [], flags = () => ({})) {
|
|
8
8
|
return makeOpenCliSkill(label, (i) => opencliCommand(base, positional(i), flags(i)));
|
|
9
9
|
}
|
|
10
|
+
/** Same shape as openSkill but routes to the bnbot CLI instead of opencli.
|
|
11
|
+
* Use for any site whose scraper is implemented natively in bnbot (zero
|
|
12
|
+
* opencli dependency) — e.g. douban, whose opencli path wedges on anti-bot
|
|
13
|
+
* pages while `bnbot douban …` works against the logged-in browser. */
|
|
14
|
+
function bnbotSkill(label, base, positional = () => [], flags = () => ({})) {
|
|
15
|
+
return makeOpenCliSkill(label, (i) => bnbotCommand(base, positional(i), flags(i)));
|
|
16
|
+
}
|
|
10
17
|
export const ggSearchSkill = makeOpenCliSkill("google search", (i) => bnbotCommand(["google", "search"], [query(i)], { limit: limit(i), lang: str(i, ["lang", "language"]) }));
|
|
11
18
|
export const ggSuggestSkill = makeOpenCliSkill("google suggest", (i) => bnbotCommand(["google", "suggest"], [query(i)], { limit: limit(i), lang: str(i, ["lang", "language"]) }));
|
|
12
19
|
export const ggNewsSkill = makeOpenCliSkill("google news", (i) => bnbotCommand(["google", "news"], str(i, ["query", "keyword", "q"]) ? [str(i, ["query", "keyword", "q"])] : [], {
|
|
@@ -113,22 +120,24 @@ export const kr36SearchSkill = openSkill("36kr search", ["36kr", "search"], (i)
|
|
|
113
120
|
export const kr36ArticleSkill = openSkill("36kr article", ["36kr", "article"], (i) => [
|
|
114
121
|
reqStr(i, ["id", "url"], "id"),
|
|
115
122
|
]);
|
|
116
|
-
export const dbSearchSkill =
|
|
123
|
+
export const dbSearchSkill = bnbotSkill("douban search", ["douban", "search"], (i) => [query(i)], (i) => ({
|
|
117
124
|
limit: limit(i),
|
|
118
125
|
type: str(i, ["type"]),
|
|
119
126
|
}));
|
|
120
|
-
export const dbMovieHotSkill =
|
|
127
|
+
export const dbMovieHotSkill = bnbotSkill("douban movie hot", ["douban", "movie-hot"], () => [], (i) => ({
|
|
128
|
+
limit: limit(i),
|
|
129
|
+
}));
|
|
130
|
+
export const dbBookHotSkill = bnbotSkill("douban book hot", ["douban", "book-hot"], () => [], (i) => ({
|
|
121
131
|
limit: limit(i),
|
|
122
132
|
}));
|
|
123
|
-
export const
|
|
133
|
+
export const dbTop250Skill = bnbotSkill("douban top250", ["douban", "top250"], () => [], (i) => ({
|
|
124
134
|
limit: limit(i),
|
|
125
135
|
}));
|
|
126
|
-
export const
|
|
127
|
-
export const dbPhotosSkill = openSkill("douban photos", ["douban", "photos"], (i) => [id(i)], (i) => ({
|
|
136
|
+
export const dbPhotosSkill = bnbotSkill("douban photos", ["douban", "photos"], (i) => [id(i)], (i) => ({
|
|
128
137
|
limit: limit(i),
|
|
129
138
|
type: str(i, ["type"]),
|
|
130
139
|
}));
|
|
131
|
-
export const sfNewsSkill =
|
|
140
|
+
export const sfNewsSkill = bnbotSkill("sinafinance news", ["sinafinance", "news"], () => [], (i) => ({
|
|
132
141
|
limit: limit(i),
|
|
133
142
|
type: str(i, ["type"]),
|
|
134
143
|
}));
|
|
@@ -137,13 +146,13 @@ export const sfStockSkill = openSkill("sinafinance stock", ["sinafinance", "stoc
|
|
|
137
146
|
reqStr(i, ["key", "query", "symbol", "ticker"], "key"),
|
|
138
147
|
], (i) => ({ market: str(i, ["market"]) }));
|
|
139
148
|
export const jkFeedSkill = openSkill("jike feed", ["jike", "feed"], () => [], (i) => ({ limit: limit(i) }));
|
|
140
|
-
export const jkSearchSkill =
|
|
149
|
+
export const jkSearchSkill = bnbotSkill("jike search", ["jike", "search"], (i) => [query(i)], (i) => ({
|
|
141
150
|
limit: limit(i),
|
|
142
151
|
}));
|
|
143
|
-
export const xqSearchSkill =
|
|
152
|
+
export const xqSearchSkill = bnbotSkill("xueqiu search", ["xueqiu", "search"], (i) => [query(i)], (i) => ({
|
|
144
153
|
limit: limit(i),
|
|
145
154
|
}));
|
|
146
|
-
export const xqHotSkill =
|
|
155
|
+
export const xqHotSkill = bnbotSkill("xueqiu hot", ["xueqiu", "hot"], () => [], (i) => ({ limit: limit(i) }));
|
|
147
156
|
export const xqHotStockSkill = openSkill("xueqiu hot stock", ["xueqiu", "hot-stock"], () => [], (i) => ({
|
|
148
157
|
limit: limit(i),
|
|
149
158
|
type: str(i, ["type"]),
|
|
@@ -183,23 +192,17 @@ export const wrRankingSkill = openSkill("weread ranking", ["weread", "ranking"],
|
|
|
183
192
|
export const wrBookSkill = openSkill("weread book", ["weread", "book"], (i) => [
|
|
184
193
|
reqStr(i, ["book_id", "bookId", "id"], "book_id"),
|
|
185
194
|
]);
|
|
186
|
-
export const ctSearchSkill =
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
export const ctHotelSuggestSkill = openSkill("ctrip hotel suggest", ["ctrip", "hotel-suggest"], (i) => [query(i)], (i) => ({
|
|
190
|
-
limit: limit(i),
|
|
191
|
-
}));
|
|
192
|
-
export const ctHotelSearchSkill = openSkill("ctrip hotel search", ["ctrip", "hotel-search"], (i) => [
|
|
193
|
-
reqStr(i, ["city", "city_id", "cityId"], "city"),
|
|
194
|
-
], (i) => ({
|
|
195
|
+
export const ctSearchSkill = makeOpenCliSkill("ctrip search", (i) => bnbotCommand(["ctrip", "search"], [query(i)], { limit: limit(i) }));
|
|
196
|
+
export const ctHotelSuggestSkill = makeOpenCliSkill("ctrip hotel suggest", (i) => bnbotCommand(["ctrip", "hotel-suggest"], [query(i)], { limit: limit(i) }));
|
|
197
|
+
export const ctHotelSearchSkill = makeOpenCliSkill("ctrip hotel search", (i) => bnbotCommand(["ctrip", "hotel-search"], [reqStr(i, ["city", "city_id", "cityId"], "city")], {
|
|
195
198
|
checkin: reqStr(i, ["checkin", "check_in", "checkIn"], "checkin"),
|
|
196
199
|
checkout: reqStr(i, ["checkout", "check_out", "checkOut"], "checkout"),
|
|
197
200
|
limit: limit(i),
|
|
198
201
|
}));
|
|
199
|
-
export const ctFlightSkill =
|
|
202
|
+
export const ctFlightSkill = makeOpenCliSkill("ctrip flight", (i) => bnbotCommand(["ctrip", "flight"], [
|
|
200
203
|
reqStr(i, ["from", "from_iata", "fromIata"], "from"),
|
|
201
204
|
reqStr(i, ["to", "to_iata", "toIata"], "to"),
|
|
202
|
-
],
|
|
205
|
+
], {
|
|
203
206
|
date: reqStr(i, ["date", "departure_date", "departureDate"], "date"),
|
|
204
207
|
limit: limit(i),
|
|
205
208
|
}));
|
|
@@ -67,6 +67,8 @@ export declare function userTweetsToTwitter283(tweets: BnbotTweet[], cursor?: {
|
|
|
67
67
|
export declare function tweetConversationToTwitter283(head: BnbotTweet, replies?: BnbotTweet[]): unknown;
|
|
68
68
|
export interface BnbotTrend {
|
|
69
69
|
name?: string;
|
|
70
|
+
topic?: string;
|
|
71
|
+
tweets?: string | number;
|
|
70
72
|
tweet_volume?: number | null;
|
|
71
73
|
url?: string;
|
|
72
74
|
category?: string;
|
|
@@ -136,8 +136,12 @@ export function trendsToTwitter283(trends) {
|
|
|
136
136
|
return {
|
|
137
137
|
data: {
|
|
138
138
|
trends: trends.map((t) => ({
|
|
139
|
-
name: t.name ?? "",
|
|
140
|
-
tweet_volume: typeof t.tweet_volume === "number"
|
|
139
|
+
name: t.name ?? t.topic ?? "",
|
|
140
|
+
tweet_volume: typeof t.tweet_volume === "number"
|
|
141
|
+
? t.tweet_volume
|
|
142
|
+
: typeof t.tweets === "number"
|
|
143
|
+
? t.tweets
|
|
144
|
+
: null,
|
|
141
145
|
url: t.url ?? "",
|
|
142
146
|
...(t.category ? { category: t.category } : {}),
|
|
143
147
|
})),
|