@jackwener/opencli 1.0.1 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/build-extension.yml +62 -0
- package/.github/workflows/ci.yml +6 -6
- package/.github/workflows/e2e-headed.yml +2 -2
- package/.github/workflows/pkg-pr-new.yml +2 -2
- package/.github/workflows/release.yml +2 -5
- package/.github/workflows/security.yml +2 -2
- package/CDP.md +1 -1
- package/CDP.zh-CN.md +1 -1
- package/README.md +15 -7
- package/README.zh-CN.md +15 -7
- package/SKILL.md +3 -5
- package/dist/browser/cdp.d.ts +27 -0
- package/dist/browser/cdp.js +295 -0
- package/dist/browser/index.d.ts +3 -0
- package/dist/browser/index.js +4 -0
- package/dist/browser/page.js +2 -23
- package/dist/browser/utils.d.ts +10 -0
- package/dist/browser/utils.js +27 -0
- package/dist/browser.test.js +42 -1
- package/dist/chaoxing.d.ts +58 -0
- package/dist/chaoxing.js +225 -0
- package/dist/chaoxing.test.d.ts +1 -0
- package/dist/chaoxing.test.js +38 -0
- package/dist/cli-manifest.json +203 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +197 -0
- package/dist/clis/boss/chatlist.d.ts +1 -0
- package/dist/clis/boss/chatlist.js +50 -0
- package/dist/clis/boss/chatmsg.d.ts +1 -0
- package/dist/clis/boss/chatmsg.js +73 -0
- package/dist/clis/boss/send.d.ts +1 -0
- package/dist/clis/boss/send.js +176 -0
- package/dist/clis/chaoxing/assignments.d.ts +1 -0
- package/dist/clis/chaoxing/assignments.js +74 -0
- package/dist/clis/chaoxing/exams.d.ts +1 -0
- package/dist/clis/chaoxing/exams.js +74 -0
- package/dist/clis/chatgpt/ask.js +15 -14
- package/dist/clis/chatgpt/ax.d.ts +1 -0
- package/dist/clis/chatgpt/ax.js +78 -0
- package/dist/clis/chatgpt/read.js +5 -6
- package/dist/clis/twitter/post.js +9 -2
- package/dist/clis/twitter/search.js +14 -33
- package/dist/clis/xiaohongshu/download.d.ts +1 -1
- package/dist/clis/xiaohongshu/download.js +1 -1
- package/dist/engine.js +24 -13
- package/dist/explore.js +46 -101
- package/dist/main.js +4 -193
- package/dist/output.d.ts +1 -1
- package/dist/registry.d.ts +3 -3
- package/dist/scripts/framework.d.ts +4 -0
- package/dist/scripts/framework.js +21 -0
- package/dist/scripts/interact.d.ts +4 -0
- package/dist/scripts/interact.js +20 -0
- package/dist/scripts/store.d.ts +9 -0
- package/dist/scripts/store.js +44 -0
- package/dist/synthesize.js +1 -1
- package/extension/dist/background.js +338 -430
- package/extension/manifest.json +2 -2
- package/extension/src/background.ts +2 -2
- package/package.json +1 -1
- package/src/browser/cdp.ts +295 -0
- package/src/browser/index.ts +4 -0
- package/src/browser/page.ts +2 -24
- package/src/browser/utils.ts +27 -0
- package/src/browser.test.ts +46 -0
- package/src/chaoxing.test.ts +45 -0
- package/src/chaoxing.ts +268 -0
- package/src/cli.ts +185 -0
- package/src/clis/antigravity/SKILL.md +5 -0
- package/src/clis/boss/chatlist.ts +50 -0
- package/src/clis/boss/chatmsg.ts +70 -0
- package/src/clis/boss/send.ts +193 -0
- package/src/clis/chaoxing/README.md +36 -0
- package/src/clis/chaoxing/README.zh-CN.md +35 -0
- package/src/clis/chaoxing/assignments.ts +88 -0
- package/src/clis/chaoxing/exams.ts +88 -0
- package/src/clis/chatgpt/ask.ts +14 -15
- package/src/clis/chatgpt/ax.ts +81 -0
- package/src/clis/chatgpt/read.ts +5 -7
- package/src/clis/twitter/post.ts +9 -2
- package/src/clis/twitter/search.ts +15 -33
- package/src/clis/xiaohongshu/download.ts +1 -1
- package/src/engine.ts +20 -13
- package/src/explore.ts +51 -100
- package/src/main.ts +4 -180
- package/src/output.ts +12 -12
- package/src/registry.ts +3 -3
- package/src/scripts/framework.ts +20 -0
- package/src/scripts/interact.ts +22 -0
- package/src/scripts/store.ts +40 -0
- package/src/synthesize.ts +1 -1
package/dist/clis/chatgpt/ask.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { execSync, spawnSync } from 'node:child_process';
|
|
2
2
|
import { cli, Strategy } from '../../registry.js';
|
|
3
|
+
import { getVisibleChatMessages } from './ax.js';
|
|
3
4
|
export const askCommand = cli({
|
|
4
5
|
site: 'chatgpt',
|
|
5
6
|
name: 'ask',
|
|
@@ -21,6 +22,7 @@ export const askCommand = cli({
|
|
|
21
22
|
clipBackup = execSync('pbpaste', { encoding: 'utf-8' });
|
|
22
23
|
}
|
|
23
24
|
catch { }
|
|
25
|
+
const messagesBefore = getVisibleChatMessages();
|
|
24
26
|
// Send the message
|
|
25
27
|
spawnSync('pbcopy', { input: text });
|
|
26
28
|
execSync("osascript -e 'tell application \"ChatGPT\" to activate'");
|
|
@@ -32,28 +34,27 @@ export const askCommand = cli({
|
|
|
32
34
|
"-e 'keystroke return' " +
|
|
33
35
|
"-e 'end tell'";
|
|
34
36
|
execSync(cmd);
|
|
35
|
-
//
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
37
|
+
// Restore clipboard after the prompt is sent.
|
|
38
|
+
if (clipBackup)
|
|
39
|
+
spawnSync('pbcopy', { input: clipBackup });
|
|
40
|
+
// Wait for response, then read the latest visible assistant message from the AX tree.
|
|
41
|
+
const pollInterval = 1;
|
|
39
42
|
const maxPolls = Math.ceil(timeout / pollInterval);
|
|
40
43
|
let response = '';
|
|
41
44
|
for (let i = 0; i < maxPolls; i++) {
|
|
42
|
-
// Wait
|
|
43
45
|
execSync(`sleep ${pollInterval}`);
|
|
44
|
-
// Try Cmd+Shift+C to copy the latest response
|
|
45
46
|
execSync("osascript -e 'tell application \"ChatGPT\" to activate'");
|
|
46
|
-
execSync("osascript -e '
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
47
|
+
execSync("osascript -e 'delay 0.2'");
|
|
48
|
+
const messagesNow = getVisibleChatMessages();
|
|
49
|
+
if (messagesNow.length <= messagesBefore.length)
|
|
50
|
+
continue;
|
|
51
|
+
const newMessages = messagesNow.slice(messagesBefore.length);
|
|
52
|
+
const candidate = [...newMessages].reverse().find((message) => message !== text);
|
|
53
|
+
if (candidate) {
|
|
54
|
+
response = candidate;
|
|
51
55
|
break;
|
|
52
56
|
}
|
|
53
57
|
}
|
|
54
|
-
// Restore clipboard
|
|
55
|
-
if (clipBackup)
|
|
56
|
-
spawnSync('pbcopy', { input: clipBackup });
|
|
57
58
|
if (!response) {
|
|
58
59
|
return [
|
|
59
60
|
{ Role: 'User', Text: text },
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function getVisibleChatMessages(): string[];
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { execFileSync } from 'node:child_process';
|
|
2
|
+
const AX_READ_SCRIPT = `
|
|
3
|
+
import Cocoa
|
|
4
|
+
import ApplicationServices
|
|
5
|
+
|
|
6
|
+
func attr(_ el: AXUIElement, _ name: String) -> AnyObject? {
|
|
7
|
+
var value: CFTypeRef?
|
|
8
|
+
guard AXUIElementCopyAttributeValue(el, name as CFString, &value) == .success else { return nil }
|
|
9
|
+
return value as AnyObject?
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
func s(_ el: AXUIElement, _ name: String) -> String? {
|
|
13
|
+
if let v = attr(el, name) as? String, !v.isEmpty { return v }
|
|
14
|
+
return nil
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
func children(_ el: AXUIElement) -> [AXUIElement] {
|
|
18
|
+
(attr(el, kAXChildrenAttribute as String) as? [AnyObject] ?? []).map { $0 as! AXUIElement }
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
func collectLists(_ el: AXUIElement, into out: inout [AXUIElement]) {
|
|
22
|
+
let role = s(el, kAXRoleAttribute as String) ?? ""
|
|
23
|
+
if role == kAXListRole as String { out.append(el) }
|
|
24
|
+
for c in children(el) { collectLists(c, into: &out) }
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
func collectTexts(_ el: AXUIElement, into out: inout [String]) {
|
|
28
|
+
let role = s(el, kAXRoleAttribute as String) ?? ""
|
|
29
|
+
if role == kAXStaticTextRole as String {
|
|
30
|
+
if let text = s(el, kAXDescriptionAttribute as String), !text.isEmpty {
|
|
31
|
+
out.append(text)
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
for c in children(el) { collectTexts(c, into: &out) }
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
guard let app = NSRunningApplication.runningApplications(withBundleIdentifier: "com.openai.chat").first else {
|
|
38
|
+
fputs("ChatGPT not running\\n", stderr)
|
|
39
|
+
exit(1)
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
let axApp = AXUIElementCreateApplication(app.processIdentifier)
|
|
43
|
+
guard let win = attr(axApp, kAXFocusedWindowAttribute as String) as! AXUIElement? else {
|
|
44
|
+
fputs("No focused ChatGPT window\\n", stderr)
|
|
45
|
+
exit(1)
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
var lists: [AXUIElement] = []
|
|
49
|
+
collectLists(win, into: &lists)
|
|
50
|
+
|
|
51
|
+
var best: [String] = []
|
|
52
|
+
for list in lists {
|
|
53
|
+
var texts: [String] = []
|
|
54
|
+
collectTexts(list, into: &texts)
|
|
55
|
+
if texts.count > best.count {
|
|
56
|
+
best = texts
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
let data = try! JSONSerialization.data(withJSONObject: best, options: [])
|
|
61
|
+
print(String(data: data, encoding: .utf8)!)
|
|
62
|
+
`;
|
|
63
|
+
export function getVisibleChatMessages() {
|
|
64
|
+
const output = execFileSync('swift', ['-'], {
|
|
65
|
+
input: AX_READ_SCRIPT,
|
|
66
|
+
encoding: 'utf-8',
|
|
67
|
+
maxBuffer: 10 * 1024 * 1024,
|
|
68
|
+
}).trim();
|
|
69
|
+
if (!output)
|
|
70
|
+
return [];
|
|
71
|
+
const parsed = JSON.parse(output);
|
|
72
|
+
if (!Array.isArray(parsed))
|
|
73
|
+
return [];
|
|
74
|
+
return parsed
|
|
75
|
+
.filter((item) => typeof item === 'string')
|
|
76
|
+
.map((item) => item.replace(/[\uFFFC\u200B-\u200D\uFEFF]/g, '').trim())
|
|
77
|
+
.filter((item) => item.length > 0);
|
|
78
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { execSync } from 'node:child_process';
|
|
2
2
|
import { cli, Strategy } from '../../registry.js';
|
|
3
|
+
import { getVisibleChatMessages } from './ax.js';
|
|
3
4
|
export const readCommand = cli({
|
|
4
5
|
site: 'chatgpt',
|
|
5
6
|
name: 'read',
|
|
@@ -12,14 +13,12 @@ export const readCommand = cli({
|
|
|
12
13
|
func: async (page) => {
|
|
13
14
|
try {
|
|
14
15
|
execSync("osascript -e 'tell application \"ChatGPT\" to activate'");
|
|
15
|
-
execSync("osascript -e 'delay 0.5'");
|
|
16
|
-
execSync("osascript -e 'tell application \"System Events\" to keystroke \"c\" using {command down, shift down}'");
|
|
17
16
|
execSync("osascript -e 'delay 0.3'");
|
|
18
|
-
const
|
|
19
|
-
if (!
|
|
20
|
-
return [{ Role: 'System', Text: 'No
|
|
17
|
+
const messages = getVisibleChatMessages();
|
|
18
|
+
if (!messages.length) {
|
|
19
|
+
return [{ Role: 'System', Text: 'No visible chat messages were found in the current ChatGPT window.' }];
|
|
21
20
|
}
|
|
22
|
-
return [{ Role: 'Assistant', Text:
|
|
21
|
+
return [{ Role: 'Assistant', Text: messages[messages.length - 1] }];
|
|
23
22
|
}
|
|
24
23
|
catch (err) {
|
|
25
24
|
throw new Error("Failed to read from ChatGPT: " + err.message);
|
|
@@ -23,8 +23,15 @@ cli({
|
|
|
23
23
|
const box = document.querySelector('[data-testid="tweetTextarea_0"]');
|
|
24
24
|
if (box) {
|
|
25
25
|
box.focus();
|
|
26
|
-
//
|
|
27
|
-
|
|
26
|
+
// Simulate a paste event to properly handle newlines in Draft.js/React
|
|
27
|
+
const textToInsert = ${JSON.stringify(kwargs.text)};
|
|
28
|
+
const dataTransfer = new DataTransfer();
|
|
29
|
+
dataTransfer.setData('text/plain', textToInsert);
|
|
30
|
+
box.dispatchEvent(new ClipboardEvent('paste', {
|
|
31
|
+
clipboardData: dataTransfer,
|
|
32
|
+
bubbles: true,
|
|
33
|
+
cancelable: true
|
|
34
|
+
}));
|
|
28
35
|
} else {
|
|
29
36
|
return { ok: false, message: 'Could not find the tweet composer text area.' };
|
|
30
37
|
}
|
|
@@ -20,45 +20,26 @@ cli({
|
|
|
20
20
|
// SPA navigation preserves the JS context, so the monkey-patched
|
|
21
21
|
// fetch will capture the SearchTimeline API call.
|
|
22
22
|
await page.installInterceptor('SearchTimeline');
|
|
23
|
-
// 3.
|
|
24
|
-
//
|
|
23
|
+
// 3. Trigger SPA navigation to search results via history API.
|
|
24
|
+
// pushState + popstate triggers React Router's listener without
|
|
25
|
+
// a full page reload, so the interceptor stays alive.
|
|
26
|
+
// Note: the previous approach (nativeSetter + Enter keydown on the
|
|
27
|
+
// search input) does not reliably trigger Twitter's form submission.
|
|
28
|
+
const searchUrl = JSON.stringify(`/search?q=${encodeURIComponent(query)}&f=top`);
|
|
25
29
|
await page.evaluate(`
|
|
26
30
|
(() => {
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
input.focus();
|
|
30
|
-
const nativeSetter = Object.getOwnPropertyDescriptor(
|
|
31
|
-
HTMLInputElement.prototype, 'value'
|
|
32
|
-
).set;
|
|
33
|
-
nativeSetter.call(input, ${JSON.stringify(query)});
|
|
34
|
-
input.dispatchEvent(new Event('input', { bubbles: true }));
|
|
35
|
-
})()
|
|
36
|
-
`);
|
|
37
|
-
await page.wait(0.5);
|
|
38
|
-
// Press Enter to submit
|
|
39
|
-
await page.evaluate(`
|
|
40
|
-
(() => {
|
|
41
|
-
const input = document.querySelector('input[data-testid="SearchBox_Search_Input"]');
|
|
42
|
-
if (!input) throw new Error('Search input not found');
|
|
43
|
-
input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', code: 'Enter', keyCode: 13, bubbles: true }));
|
|
31
|
+
window.history.pushState({}, '', ${searchUrl});
|
|
32
|
+
window.dispatchEvent(new PopStateEvent('popstate', { state: {} }));
|
|
44
33
|
})()
|
|
45
34
|
`);
|
|
46
35
|
await page.wait(5);
|
|
47
|
-
//
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
const tabs = document.querySelectorAll('[role="tab"]');
|
|
52
|
-
for (const tab of tabs) {
|
|
53
|
-
if (tab.textContent.trim() === 'Top') { tab.click(); break; }
|
|
54
|
-
}
|
|
55
|
-
})()
|
|
56
|
-
`);
|
|
57
|
-
await page.wait(2);
|
|
36
|
+
// Verify SPA navigation succeeded
|
|
37
|
+
const currentPath = await page.evaluate('() => window.location.pathname');
|
|
38
|
+
if (!currentPath?.startsWith('/search')) {
|
|
39
|
+
throw new Error('SPA navigation to /search failed. Twitter may have changed its routing.');
|
|
58
40
|
}
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
await page.autoScroll({ times: 2, delayMs: 2000 });
|
|
41
|
+
// 4. Scroll to trigger additional pagination
|
|
42
|
+
await page.autoScroll({ times: 3, delayMs: 2000 });
|
|
62
43
|
// 6. Retrieve captured data
|
|
63
44
|
const requests = await page.getInterceptedRequests();
|
|
64
45
|
if (!requests || requests.length === 0)
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Xiaohongshu download — download images and videos from a note.
|
|
3
3
|
*
|
|
4
4
|
* Usage:
|
|
5
|
-
* opencli xiaohongshu download --
|
|
5
|
+
* opencli xiaohongshu download --note_id abc123 --output ./xhs
|
|
6
6
|
*/
|
|
7
7
|
import * as fs from 'node:fs';
|
|
8
8
|
import * as path from 'node:path';
|
package/dist/engine.js
CHANGED
|
@@ -24,12 +24,15 @@ export async function discoverClis(...dirs) {
|
|
|
24
24
|
// Fast path: try manifest first (production / post-build)
|
|
25
25
|
for (const dir of dirs) {
|
|
26
26
|
const manifestPath = path.resolve(dir, '..', 'cli-manifest.json');
|
|
27
|
-
|
|
28
|
-
|
|
27
|
+
try {
|
|
28
|
+
await fs.promises.access(manifestPath);
|
|
29
|
+
await loadFromManifest(manifestPath, dir);
|
|
29
30
|
continue; // Skip filesystem scan for this directory
|
|
30
31
|
}
|
|
31
|
-
|
|
32
|
-
|
|
32
|
+
catch {
|
|
33
|
+
// Fallback: runtime filesystem scan (development)
|
|
34
|
+
await discoverClisFromFs(dir);
|
|
35
|
+
}
|
|
33
36
|
}
|
|
34
37
|
}
|
|
35
38
|
/**
|
|
@@ -37,9 +40,10 @@ export async function discoverClis(...dirs) {
|
|
|
37
40
|
* YAML pipelines are inlined — zero YAML parsing at runtime.
|
|
38
41
|
* TS modules are deferred — loaded lazily on first execution.
|
|
39
42
|
*/
|
|
40
|
-
function loadFromManifest(manifestPath, clisDir) {
|
|
43
|
+
async function loadFromManifest(manifestPath, clisDir) {
|
|
41
44
|
try {
|
|
42
|
-
const
|
|
45
|
+
const raw = await fs.promises.readFile(manifestPath, 'utf-8');
|
|
46
|
+
const manifest = JSON.parse(raw);
|
|
43
47
|
for (const entry of manifest) {
|
|
44
48
|
if (entry.type === 'yaml') {
|
|
45
49
|
// YAML pipelines fully inlined in manifest — register directly
|
|
@@ -90,17 +94,24 @@ function loadFromManifest(manifestPath, clisDir) {
|
|
|
90
94
|
* Fallback: traditional filesystem scan (used during development with tsx).
|
|
91
95
|
*/
|
|
92
96
|
async function discoverClisFromFs(dir) {
|
|
93
|
-
|
|
97
|
+
try {
|
|
98
|
+
await fs.promises.access(dir);
|
|
99
|
+
}
|
|
100
|
+
catch {
|
|
94
101
|
return;
|
|
102
|
+
}
|
|
95
103
|
const promises = [];
|
|
96
|
-
|
|
104
|
+
const sites = await fs.promises.readdir(dir);
|
|
105
|
+
for (const site of sites) {
|
|
97
106
|
const siteDir = path.join(dir, site);
|
|
98
|
-
|
|
107
|
+
const stat = await fs.promises.stat(siteDir);
|
|
108
|
+
if (!stat.isDirectory())
|
|
99
109
|
continue;
|
|
100
|
-
|
|
110
|
+
const files = await fs.promises.readdir(siteDir);
|
|
111
|
+
for (const file of files) {
|
|
101
112
|
const filePath = path.join(siteDir, file);
|
|
102
113
|
if (file.endsWith('.yaml') || file.endsWith('.yml')) {
|
|
103
|
-
registerYamlCli(filePath, site);
|
|
114
|
+
promises.push(registerYamlCli(filePath, site));
|
|
104
115
|
}
|
|
105
116
|
else if ((file.endsWith('.js') && !file.endsWith('.d.js')) ||
|
|
106
117
|
(file.endsWith('.ts') && !file.endsWith('.d.ts') && !file.endsWith('.test.ts'))) {
|
|
@@ -112,9 +123,9 @@ async function discoverClisFromFs(dir) {
|
|
|
112
123
|
}
|
|
113
124
|
await Promise.all(promises);
|
|
114
125
|
}
|
|
115
|
-
function registerYamlCli(filePath, defaultSite) {
|
|
126
|
+
async function registerYamlCli(filePath, defaultSite) {
|
|
116
127
|
try {
|
|
117
|
-
const raw = fs.
|
|
128
|
+
const raw = await fs.promises.readFile(filePath, 'utf-8');
|
|
118
129
|
const def = yaml.load(raw);
|
|
119
130
|
if (!def || typeof def !== 'object')
|
|
120
131
|
return;
|
package/dist/explore.js
CHANGED
|
@@ -9,6 +9,9 @@ import * as fs from 'node:fs';
|
|
|
9
9
|
import * as path from 'node:path';
|
|
10
10
|
import { DEFAULT_BROWSER_EXPLORE_TIMEOUT, browserSession, runWithTimeout } from './runtime.js';
|
|
11
11
|
import { VOLATILE_PARAMS, SEARCH_PARAMS, PAGINATION_PARAMS, LIMIT_PARAMS, FIELD_ROLES } from './constants.js';
|
|
12
|
+
import { detectFramework } from './scripts/framework.js';
|
|
13
|
+
import { discoverStores } from './scripts/store.js';
|
|
14
|
+
import { interactFuzz } from './scripts/interact.js';
|
|
12
15
|
// ── Site name detection ────────────────────────────────────────────────────
|
|
13
16
|
const KNOWN_SITE_ALIASES = {
|
|
14
17
|
'x.com': 'twitter', 'twitter.com': 'twitter',
|
|
@@ -134,11 +137,13 @@ function flattenFields(obj, prefix, maxDepth) {
|
|
|
134
137
|
if (maxDepth <= 0 || !obj || typeof obj !== 'object')
|
|
135
138
|
return [];
|
|
136
139
|
const names = [];
|
|
137
|
-
|
|
140
|
+
const record = obj;
|
|
141
|
+
for (const key of Object.keys(record)) {
|
|
138
142
|
const full = prefix ? `${prefix}.${key}` : key;
|
|
139
143
|
names.push(full);
|
|
140
|
-
|
|
141
|
-
|
|
144
|
+
const val = record[key];
|
|
145
|
+
if (val && typeof val === 'object' && !Array.isArray(val))
|
|
146
|
+
names.push(...flattenFields(val, full, maxDepth - 1));
|
|
142
147
|
}
|
|
143
148
|
return names;
|
|
144
149
|
}
|
|
@@ -200,83 +205,11 @@ function inferStrategy(authIndicators) {
|
|
|
200
205
|
return 'cookie';
|
|
201
206
|
}
|
|
202
207
|
// ── Framework detection ────────────────────────────────────────────────────
|
|
203
|
-
const FRAMEWORK_DETECT_JS =
|
|
204
|
-
() => {
|
|
205
|
-
const r = {};
|
|
206
|
-
try {
|
|
207
|
-
const app = document.querySelector('#app');
|
|
208
|
-
r.vue3 = !!(app && app.__vue_app__);
|
|
209
|
-
r.vue2 = !!(app && app.__vue__);
|
|
210
|
-
r.react = !!window.__REACT_DEVTOOLS_GLOBAL_HOOK__ || !!document.querySelector('[data-reactroot]');
|
|
211
|
-
r.nextjs = !!window.__NEXT_DATA__;
|
|
212
|
-
r.nuxt = !!window.__NUXT__;
|
|
213
|
-
if (r.vue3 && app.__vue_app__) { const gp = app.__vue_app__.config?.globalProperties; r.pinia = !!(gp && gp.$pinia); r.vuex = !!(gp && gp.$store); }
|
|
214
|
-
} catch {}
|
|
215
|
-
return r;
|
|
216
|
-
}
|
|
217
|
-
`;
|
|
208
|
+
const FRAMEWORK_DETECT_JS = detectFramework.toString();
|
|
218
209
|
// ── Store discovery ────────────────────────────────────────────────────────
|
|
219
|
-
const STORE_DISCOVER_JS =
|
|
220
|
-
() => {
|
|
221
|
-
const stores = [];
|
|
222
|
-
try {
|
|
223
|
-
const app = document.querySelector('#app');
|
|
224
|
-
if (!app?.__vue_app__) return stores;
|
|
225
|
-
const gp = app.__vue_app__.config?.globalProperties;
|
|
226
|
-
|
|
227
|
-
// Pinia stores
|
|
228
|
-
const pinia = gp?.$pinia;
|
|
229
|
-
if (pinia?._s) {
|
|
230
|
-
pinia._s.forEach((store, id) => {
|
|
231
|
-
const actions = [];
|
|
232
|
-
const stateKeys = [];
|
|
233
|
-
for (const k in store) {
|
|
234
|
-
try {
|
|
235
|
-
if (k.startsWith('$') || k.startsWith('_')) continue;
|
|
236
|
-
if (typeof store[k] === 'function') actions.push(k);
|
|
237
|
-
else stateKeys.push(k);
|
|
238
|
-
} catch {}
|
|
239
|
-
}
|
|
240
|
-
stores.push({ type: 'pinia', id, actions: actions.slice(0, 20), stateKeys: stateKeys.slice(0, 15) });
|
|
241
|
-
});
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
// Vuex store modules
|
|
245
|
-
const vuex = gp?.$store;
|
|
246
|
-
if (vuex?._modules?.root?._children) {
|
|
247
|
-
const children = vuex._modules.root._children;
|
|
248
|
-
for (const [modName, mod] of Object.entries(children)) {
|
|
249
|
-
const actions = Object.keys(mod._rawModule?.actions ?? {}).slice(0, 20);
|
|
250
|
-
const stateKeys = Object.keys(mod.state ?? {}).slice(0, 15);
|
|
251
|
-
stores.push({ type: 'vuex', id: modName, actions, stateKeys });
|
|
252
|
-
}
|
|
253
|
-
}
|
|
254
|
-
} catch {}
|
|
255
|
-
return stores;
|
|
256
|
-
}
|
|
257
|
-
`;
|
|
210
|
+
const STORE_DISCOVER_JS = discoverStores.toString();
|
|
258
211
|
// ── Auto-Interaction (Fuzzing) ─────────────────────────────────────────────
|
|
259
|
-
const INTERACT_FUZZ_JS =
|
|
260
|
-
async () => {
|
|
261
|
-
const sleep = ms => new Promise(r => setTimeout(r, ms));
|
|
262
|
-
const clickables = Array.from(document.querySelectorAll(
|
|
263
|
-
'button, [role="button"], [role="tab"], .tab, .btn, a[href="javascript:void(0)"], a[href="#"]'
|
|
264
|
-
)).slice(0, 15); // limit to 15 to avoid endless loops
|
|
265
|
-
|
|
266
|
-
let clicked = 0;
|
|
267
|
-
for (const el of clickables) {
|
|
268
|
-
try {
|
|
269
|
-
const rect = el.getBoundingClientRect();
|
|
270
|
-
if (rect.width > 0 && rect.height > 0) {
|
|
271
|
-
el.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
|
|
272
|
-
clicked++;
|
|
273
|
-
await sleep(300); // give it time to trigger network
|
|
274
|
-
}
|
|
275
|
-
} catch {}
|
|
276
|
-
}
|
|
277
|
-
return clicked;
|
|
278
|
-
}
|
|
279
|
-
`;
|
|
212
|
+
const INTERACT_FUZZ_JS = interactFuzz.toString();
|
|
280
213
|
// ── Main explore function ──────────────────────────────────────────────────
|
|
281
214
|
export async function exploreUrl(url, opts) {
|
|
282
215
|
const waitSeconds = opts.waitSeconds ?? 3.0;
|
|
@@ -286,25 +219,19 @@ export async function exploreUrl(url, opts) {
|
|
|
286
219
|
// Step 1: Navigate
|
|
287
220
|
await page.goto(url);
|
|
288
221
|
await page.wait(waitSeconds);
|
|
289
|
-
// Step 2: Auto-scroll to trigger lazy loading
|
|
290
|
-
|
|
291
|
-
try {
|
|
292
|
-
await page.pressKey('End');
|
|
293
|
-
}
|
|
294
|
-
catch { }
|
|
295
|
-
await page.wait(1);
|
|
296
|
-
}
|
|
222
|
+
// Step 2: Auto-scroll to trigger lazy loading intelligently
|
|
223
|
+
await page.autoScroll({ times: 3, delayMs: 1500 }).catch(() => { });
|
|
297
224
|
// Step 2.5: Interactive Fuzzing (if requested)
|
|
298
225
|
if (opts.auto) {
|
|
299
226
|
try {
|
|
300
227
|
// First: targeted clicks by label (e.g. "字幕", "CC", "评论")
|
|
301
228
|
if (opts.clickLabels?.length) {
|
|
302
229
|
for (const label of opts.clickLabels) {
|
|
303
|
-
const safeLabel =
|
|
230
|
+
const safeLabel = JSON.stringify(label);
|
|
304
231
|
await page.evaluate(`
|
|
305
232
|
(() => {
|
|
306
233
|
const el = [...document.querySelectorAll('button, [role="button"], [role="tab"], a, span')]
|
|
307
|
-
.find(e => e.textContent && e.textContent.trim().includes(
|
|
234
|
+
.find(e => e.textContent && e.textContent.trim().includes(${safeLabel}));
|
|
308
235
|
if (el) el.click();
|
|
309
236
|
})()
|
|
310
237
|
`);
|
|
@@ -324,11 +251,27 @@ export async function exploreUrl(url, opts) {
|
|
|
324
251
|
// Step 4: Capture network traffic
|
|
325
252
|
const rawNetwork = await page.networkRequests(false);
|
|
326
253
|
const networkEntries = parseNetworkRequests(rawNetwork);
|
|
327
|
-
// Step 5: For JSON endpoints, re-fetch
|
|
328
|
-
const jsonEndpoints = networkEntries.filter(e => e.contentType.includes('json') && e.method === 'GET' && e.status === 200);
|
|
329
|
-
for (const ep of jsonEndpoints.slice(0,
|
|
254
|
+
// Step 5: For JSON endpoints missing a body, carefully re-fetch in-browser via a pristine iframe
|
|
255
|
+
const jsonEndpoints = networkEntries.filter(e => e.contentType.includes('json') && e.method === 'GET' && e.status === 200 && !e.responseBody);
|
|
256
|
+
for (const ep of jsonEndpoints.slice(0, 5)) {
|
|
330
257
|
try {
|
|
331
|
-
const body = await page.evaluate(`async () => {
|
|
258
|
+
const body = await page.evaluate(`async () => {
|
|
259
|
+
let iframe = null;
|
|
260
|
+
try {
|
|
261
|
+
iframe = document.createElement('iframe');
|
|
262
|
+
iframe.style.display = 'none';
|
|
263
|
+
document.body.appendChild(iframe);
|
|
264
|
+
const cleanFetch = iframe.contentWindow.fetch || window.fetch;
|
|
265
|
+
const r = await cleanFetch(${JSON.stringify(ep.url)}, { credentials: 'include' });
|
|
266
|
+
if (!r.ok) return null;
|
|
267
|
+
const d = await r.json();
|
|
268
|
+
return JSON.stringify(d).slice(0, 10000);
|
|
269
|
+
} catch {
|
|
270
|
+
return null;
|
|
271
|
+
} finally {
|
|
272
|
+
if (iframe && iframe.parentNode) iframe.parentNode.removeChild(iframe);
|
|
273
|
+
}
|
|
274
|
+
}`);
|
|
332
275
|
if (body && typeof body === 'string') {
|
|
333
276
|
try {
|
|
334
277
|
ep.responseBody = JSON.parse(body);
|
|
@@ -443,7 +386,7 @@ export async function exploreUrl(url, opts) {
|
|
|
443
386
|
const topStrategy = allAuth.has('signature') ? 'intercept' : allAuth.has('bearer') || allAuth.has('csrf') ? 'header' : allAuth.size === 0 ? 'public' : 'cookie';
|
|
444
387
|
const siteName = opts.site ?? detectSiteName(metadata.url || url);
|
|
445
388
|
const targetDir = opts.outDir ?? path.join('.opencli', 'explore', siteName);
|
|
446
|
-
fs.
|
|
389
|
+
await fs.promises.mkdir(targetDir, { recursive: true });
|
|
447
390
|
const result = {
|
|
448
391
|
site: siteName, target_url: url, final_url: metadata.url, title: metadata.title,
|
|
449
392
|
framework, stores, top_strategy: topStrategy,
|
|
@@ -452,24 +395,26 @@ export async function exploreUrl(url, opts) {
|
|
|
452
395
|
capabilities, auth_indicators: [...allAuth],
|
|
453
396
|
};
|
|
454
397
|
// Write artifacts
|
|
455
|
-
|
|
398
|
+
const writeTasks = [];
|
|
399
|
+
writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'manifest.json'), JSON.stringify({
|
|
456
400
|
site: siteName, target_url: url, final_url: metadata.url, title: metadata.title,
|
|
457
401
|
framework, stores: stores.map(s => ({ type: s.type, id: s.id, actions: s.actions })),
|
|
458
402
|
top_strategy: topStrategy, explored_at: new Date().toISOString(),
|
|
459
|
-
}, null, 2));
|
|
460
|
-
fs.
|
|
403
|
+
}, null, 2)));
|
|
404
|
+
writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'endpoints.json'), JSON.stringify(analyzedEndpoints.map(ep => ({
|
|
461
405
|
pattern: ep.pattern, method: ep.method, url: ep.url, status: ep.status,
|
|
462
406
|
contentType: ep.contentType, score: ep.score, queryParams: ep.queryParams,
|
|
463
407
|
itemPath: ep.responseAnalysis?.itemPath ?? null, itemCount: ep.responseAnalysis?.itemCount ?? 0,
|
|
464
408
|
detectedFields: ep.responseAnalysis?.detectedFields ?? {}, authIndicators: ep.authIndicators,
|
|
465
|
-
})), null, 2));
|
|
466
|
-
fs.
|
|
467
|
-
fs.
|
|
409
|
+
})), null, 2)));
|
|
410
|
+
writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'capabilities.json'), JSON.stringify(capabilities, null, 2)));
|
|
411
|
+
writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'auth.json'), JSON.stringify({
|
|
468
412
|
top_strategy: topStrategy, indicators: [...allAuth], framework,
|
|
469
|
-
}, null, 2));
|
|
413
|
+
}, null, 2)));
|
|
470
414
|
if (stores.length > 0) {
|
|
471
|
-
fs.
|
|
415
|
+
writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'stores.json'), JSON.stringify(stores, null, 2)));
|
|
472
416
|
}
|
|
417
|
+
await Promise.all(writeTasks);
|
|
473
418
|
return { ...result, out_dir: targetDir };
|
|
474
419
|
})(), { timeout: exploreTimeout, label: `Explore ${url}` });
|
|
475
420
|
});
|