@jackwener/opencli 1.8.0 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -49
- package/README.zh-CN.md +8 -52
- package/cli-manifest.json +1796 -191
- package/clis/_atlassian/shared.js +577 -0
- package/clis/_atlassian/shared.test.js +170 -0
- package/clis/bilibili/comment.js +125 -0
- package/clis/bilibili/comment.test.js +153 -0
- package/clis/bilibili/comments.js +116 -21
- package/clis/bilibili/comments.test.js +77 -18
- package/clis/bilibili/subtitle.js +76 -31
- package/clis/bilibili/subtitle.test.js +156 -9
- package/clis/bilibili/utils.js +63 -5
- package/clis/bilibili/utils.test.js +45 -1
- package/clis/chess/analyze.js +35 -0
- package/clis/chess/analyze.test.js +79 -0
- package/clis/chess/game.js +114 -0
- package/clis/chess/game.test.js +178 -0
- package/clis/chess/games.js +67 -0
- package/clis/chess/games.test.js +164 -0
- package/clis/chess/stats.js +32 -0
- package/clis/chess/stats.test.js +79 -0
- package/clis/chess/utils.js +170 -0
- package/clis/chess/utils.test.js +230 -0
- package/clis/confluence/commands.test.js +195 -0
- package/clis/confluence/create.js +39 -0
- package/clis/confluence/page.js +23 -0
- package/clis/confluence/search.js +34 -0
- package/clis/confluence/shared.js +173 -0
- package/clis/confluence/update.js +38 -0
- package/clis/douyin/hashtag.js +84 -23
- package/clis/douyin/hashtag.test.js +113 -0
- package/clis/geogebra/add-circle.js +46 -0
- package/clis/geogebra/add-line.js +35 -0
- package/clis/geogebra/add-point.js +27 -0
- package/clis/geogebra/add-polygon.js +25 -0
- package/clis/geogebra/eval.js +35 -0
- package/clis/geogebra/geogebra.test.js +175 -0
- package/clis/geogebra/hexagon.js +62 -0
- package/clis/geogebra/info.js +72 -0
- package/clis/geogebra/list.js +35 -0
- package/clis/geogebra/triangle.js +60 -0
- package/clis/geogebra/utils.js +271 -0
- package/clis/jira/attachments.js +28 -0
- package/clis/jira/commands.test.js +287 -0
- package/clis/jira/comments.js +28 -0
- package/clis/jira/issue.js +28 -0
- package/clis/jira/links.js +28 -0
- package/clis/jira/search.js +47 -0
- package/clis/jira/shared.js +256 -0
- package/clis/linkedin/job-detail.js +167 -0
- package/clis/linkedin/job-detail.test.js +38 -0
- package/clis/linkedin/jobs-preferences.js +113 -0
- package/clis/linkedin/jobs-preferences.test.js +43 -0
- package/clis/linkedin/post-analytics.js +74 -0
- package/clis/linkedin/post-analytics.test.js +40 -0
- package/clis/linkedin/posts-core.js +241 -0
- package/clis/linkedin/posts.js +22 -0
- package/clis/linkedin/posts.test.js +40 -0
- package/clis/linkedin/profile-analytics.js +104 -0
- package/clis/linkedin/profile-analytics.test.js +67 -0
- package/clis/linkedin/profile-experience.js +671 -0
- package/clis/linkedin/profile-experience.test.js +152 -0
- package/clis/linkedin/profile-projects.js +311 -0
- package/clis/linkedin/profile-projects.test.js +111 -0
- package/clis/linkedin/profile-read.js +148 -0
- package/clis/linkedin/profile-read.test.js +77 -0
- package/clis/linkedin/services-read.js +213 -0
- package/clis/linkedin/services-read.test.js +105 -0
- package/clis/linkedin/shared.js +124 -0
- package/clis/linkedin/timeline.js +14 -7
- package/clis/notebooklm/add-source.js +269 -0
- package/clis/notebooklm/add-source.test.js +97 -0
- package/clis/notebooklm/create.js +76 -0
- package/clis/notebooklm/create.test.js +58 -0
- package/clis/notebooklm/generate-audio.js +91 -0
- package/clis/notebooklm/generate-audio.test.js +63 -0
- package/clis/notebooklm/generate-slides.js +106 -0
- package/clis/notebooklm/generate-slides.test.js +75 -0
- package/clis/notebooklm/open.test.js +10 -10
- package/clis/notebooklm/rpc.js +20 -6
- package/clis/notebooklm/rpc.test.js +27 -1
- package/clis/notebooklm/utils.js +100 -24
- package/clis/notebooklm/utils.test.js +60 -1
- package/clis/notebooklm/write-note.js +103 -0
- package/clis/notebooklm/write-note.test.js +70 -0
- package/clis/pixiv/detail.js +41 -34
- package/clis/pixiv/detail.test.js +93 -0
- package/clis/pixiv/user.js +36 -31
- package/clis/pixiv/user.test.js +100 -0
- package/clis/pixiv/utils.js +56 -7
- package/clis/suno/generate.js +5 -0
- package/clis/suno/generate.test.js +9 -0
- package/clis/suno/status.js +3 -2
- package/clis/suno/utils.js +33 -24
- package/clis/suno/utils.test.js +106 -0
- package/clis/twitter/followers.js +6 -2
- package/clis/twitter/followers.test.js +19 -1
- package/clis/twitter/following.js +14 -5
- package/clis/twitter/following.test.js +29 -0
- package/clis/twitter/likes.js +12 -4
- package/clis/twitter/likes.test.js +26 -1
- package/clis/twitter/list-add.js +1 -1
- package/clis/twitter/list-remove.js +1 -1
- package/clis/twitter/notifications.js +4 -4
- package/clis/twitter/post.js +62 -4
- package/clis/twitter/post.test.js +35 -3
- package/clis/twitter/profile.js +81 -28
- package/clis/twitter/profile.test.js +113 -2
- package/clis/twitter/quote.js +9 -4
- package/clis/twitter/reply.js +13 -10
- package/clis/twitter/reply.test.js +41 -0
- package/clis/twitter/search.js +1 -1
- package/clis/twitter/search.test.js +35 -0
- package/clis/twitter/shared.js +11 -0
- package/clis/twitter/shared.test.js +37 -1
- package/clis/twitter/utils.js +53 -16
- package/clis/upwork/detail.js +132 -0
- package/clis/upwork/feed.js +109 -0
- package/clis/upwork/search.js +115 -0
- package/clis/upwork/upwork.test.js +566 -0
- package/clis/upwork/utils.js +323 -0
- package/clis/weread/book-search.js +438 -0
- package/clis/weread/book-search.test.js +242 -0
- package/clis/weread/search-regression.test.js +80 -0
- package/clis/weread/search.js +17 -2
- package/clis/xiaohongshu/creator-note-detail.js +165 -28
- package/clis/xiaohongshu/creator-note-detail.test.js +186 -37
- package/clis/xiaohongshu/creator-notes.js +251 -2
- package/clis/xiaohongshu/creator-notes.test.js +79 -2
- package/clis/xiaohongshu/download.js +97 -39
- package/clis/xiaohongshu/download.test.js +201 -0
- package/clis/zhihu/answer-comments.js +2 -21
- package/clis/zhihu/answer-detail.js +2 -31
- package/clis/zhihu/collection.js +2 -14
- package/clis/zhihu/collection.test.js +4 -3
- package/clis/zhihu/question.js +1 -9
- package/clis/zhihu/question.test.js +2 -2
- package/clis/zhihu/search.js +1 -12
- package/clis/zhihu/search.test.js +2 -2
- package/clis/zhihu/text.js +29 -0
- package/clis/zhihu/text.test.js +24 -0
- package/dist/src/browser/network-cache.js +13 -1
- package/dist/src/browser/network-cache.test.js +17 -0
- package/dist/src/download/index.js +13 -1
- package/dist/src/download/index.test.js +23 -1
- package/dist/src/download/media-download.test.js +3 -1
- package/dist/src/download/progress.js +2 -2
- package/dist/src/download/progress.test.js +12 -1
- package/dist/src/output.js +11 -1
- package/dist/src/output.test.js +6 -0
- package/dist/src/registry.js +1 -0
- package/dist/src/registry.test.js +11 -0
- package/package.json +1 -1
|
@@ -10,7 +10,9 @@ vi.mock('@jackwener/opencli/download', () => ({
|
|
|
10
10
|
formatCookieHeader: mockFormatCookieHeader,
|
|
11
11
|
}));
|
|
12
12
|
import { getRegistry } from '@jackwener/opencli/registry';
|
|
13
|
+
import { JSDOM } from 'jsdom';
|
|
13
14
|
import './download.js';
|
|
15
|
+
import { buildDownloadExtractJs } from './download.js';
|
|
14
16
|
function createPageMock(evaluateResult) {
|
|
15
17
|
return {
|
|
16
18
|
goto: vi.fn().mockResolvedValue(undefined),
|
|
@@ -113,3 +115,202 @@ describe('xiaohongshu download', () => {
|
|
|
113
115
|
expect(mockDownloadMedia).not.toHaveBeenCalled();
|
|
114
116
|
});
|
|
115
117
|
});
|
|
118
|
+
|
|
119
|
+
describe('xiaohongshu download buildDownloadExtractJs carousel ordering (JSDOM)', () => {
|
|
120
|
+
function runExtract({ html = '', initialState = null, url = 'https://www.xiaohongshu.com/explore/69f9716c000000003601f90e' } = {}) {
|
|
121
|
+
const dom = new JSDOM(`<!doctype html><html><body>${html}</body></html>`, { url, runScripts: 'outside-only' });
|
|
122
|
+
if (initialState) {
|
|
123
|
+
dom.window.__INITIAL_STATE__ = initialState;
|
|
124
|
+
}
|
|
125
|
+
const js = buildDownloadExtractJs('69f9716c000000003601f90e');
|
|
126
|
+
return dom.window.eval(js);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
it('preserves carousel order from __INITIAL_STATE__ imageList over DOM discovery order', () => {
|
|
130
|
+
const initialState = {
|
|
131
|
+
note: {
|
|
132
|
+
noteDetailMap: {
|
|
133
|
+
'69f9716c000000003601f90e': {
|
|
134
|
+
note: {
|
|
135
|
+
imageList: [
|
|
136
|
+
{ urlDefault: 'https://sns-img-bd.xhscdn.com/canonical-cover.jpg' },
|
|
137
|
+
{ urlDefault: 'https://sns-img-bd.xhscdn.com/canonical-second.jpg' },
|
|
138
|
+
{ urlDefault: 'https://sns-img-bd.xhscdn.com/canonical-third.jpg' },
|
|
139
|
+
],
|
|
140
|
+
},
|
|
141
|
+
},
|
|
142
|
+
},
|
|
143
|
+
},
|
|
144
|
+
};
|
|
145
|
+
// DOM has the same images but in a DIFFERENT order: repro for #1514.
|
|
146
|
+
const html = `
|
|
147
|
+
<div class="swiper-slide"><img src="https://sns-img-bd.xhscdn.com/canonical-second.jpg" /></div>
|
|
148
|
+
<div class="swiper-slide"><img src="https://sns-img-bd.xhscdn.com/canonical-cover.jpg" /></div>
|
|
149
|
+
<div class="swiper-slide"><img src="https://sns-img-bd.xhscdn.com/canonical-third.jpg" /></div>
|
|
150
|
+
`;
|
|
151
|
+
const result = runExtract({ html, initialState });
|
|
152
|
+
const images = result.media.filter((m) => m.type === 'image');
|
|
153
|
+
expect(images.map((m) => m.url)).toEqual([
|
|
154
|
+
'https://sns-img-bd.xhscdn.com/canonical-cover.jpg',
|
|
155
|
+
'https://sns-img-bd.xhscdn.com/canonical-second.jpg',
|
|
156
|
+
'https://sns-img-bd.xhscdn.com/canonical-third.jpg',
|
|
157
|
+
]);
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
it('prefers urlDefault but falls back to urlPre / url / infoList.WB_DFT / infoList[0]', () => {
|
|
161
|
+
const initialState = {
|
|
162
|
+
note: {
|
|
163
|
+
noteDetailMap: {
|
|
164
|
+
'69f9716c000000003601f90e': {
|
|
165
|
+
note: {
|
|
166
|
+
imageList: [
|
|
167
|
+
{ urlDefault: 'https://sns-img-bd.xhscdn.com/a.jpg' },
|
|
168
|
+
{ urlPre: 'https://sns-img-bd.xhscdn.com/b.jpg' },
|
|
169
|
+
{ url: 'https://sns-img-bd.xhscdn.com/c.jpg' },
|
|
170
|
+
{ infoList: [{ imageScene: 'WB_PRV', url: 'https://sns-img-bd.xhscdn.com/d-low.jpg' }, { imageScene: 'WB_DFT', url: 'https://sns-img-bd.xhscdn.com/d.jpg' }] },
|
|
171
|
+
{ infoList: [{ url: 'https://sns-img-bd.xhscdn.com/e.jpg' }] },
|
|
172
|
+
],
|
|
173
|
+
},
|
|
174
|
+
},
|
|
175
|
+
},
|
|
176
|
+
},
|
|
177
|
+
};
|
|
178
|
+
const result = runExtract({ initialState });
|
|
179
|
+
const urls = result.media.filter((m) => m.type === 'image').map((m) => m.url);
|
|
180
|
+
expect(urls).toEqual([
|
|
181
|
+
'https://sns-img-bd.xhscdn.com/a.jpg',
|
|
182
|
+
'https://sns-img-bd.xhscdn.com/b.jpg',
|
|
183
|
+
'https://sns-img-bd.xhscdn.com/c.jpg',
|
|
184
|
+
'https://sns-img-bd.xhscdn.com/d.jpg',
|
|
185
|
+
'https://sns-img-bd.xhscdn.com/e.jpg',
|
|
186
|
+
]);
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
it('strips imageView resize params + query strings from canonical urls', () => {
|
|
190
|
+
const initialState = {
|
|
191
|
+
note: {
|
|
192
|
+
noteDetailMap: {
|
|
193
|
+
'69f9716c000000003601f90e': {
|
|
194
|
+
note: {
|
|
195
|
+
imageList: [
|
|
196
|
+
{ urlDefault: 'https://sns-img-bd.xhscdn.com/raw/imageView2/2/w/1080/cover.jpg?expires=123' },
|
|
197
|
+
],
|
|
198
|
+
},
|
|
199
|
+
},
|
|
200
|
+
},
|
|
201
|
+
},
|
|
202
|
+
};
|
|
203
|
+
const result = runExtract({ initialState });
|
|
204
|
+
const urls = result.media.filter((m) => m.type === 'image').map((m) => m.url);
|
|
205
|
+
expect(urls).toEqual(['https://sns-img-bd.xhscdn.com/raw/cover.jpg']);
|
|
206
|
+
});
|
|
207
|
+
|
|
208
|
+
it('falls back to DOM extraction when __INITIAL_STATE__ omits imageList', () => {
|
|
209
|
+
const html = `
|
|
210
|
+
<div class="swiper-slide"><img src="https://sns-img-bd.xhscdn.com/dom-1.jpg" /></div>
|
|
211
|
+
<div class="swiper-slide"><img src="https://sns-img-bd.xhscdn.com/dom-2.jpg" /></div>
|
|
212
|
+
`;
|
|
213
|
+
const result = runExtract({ html });
|
|
214
|
+
const urls = result.media.filter((m) => m.type === 'image').map((m) => m.url);
|
|
215
|
+
expect(urls).toEqual([
|
|
216
|
+
'https://sns-img-bd.xhscdn.com/dom-1.jpg',
|
|
217
|
+
'https://sns-img-bd.xhscdn.com/dom-2.jpg',
|
|
218
|
+
]);
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
it('skips non-xhscdn / non-xiaohongshu / non-rednote urls in initial state', () => {
|
|
222
|
+
const initialState = {
|
|
223
|
+
note: {
|
|
224
|
+
noteDetailMap: {
|
|
225
|
+
'69f9716c000000003601f90e': {
|
|
226
|
+
note: {
|
|
227
|
+
imageList: [
|
|
228
|
+
{ urlDefault: 'https://sns-img-bd.xhscdn.com/keep.jpg' },
|
|
229
|
+
{ urlDefault: 'https://imgur.com/drop.jpg' },
|
|
230
|
+
{ urlDefault: '' },
|
|
231
|
+
],
|
|
232
|
+
},
|
|
233
|
+
},
|
|
234
|
+
},
|
|
235
|
+
},
|
|
236
|
+
};
|
|
237
|
+
const result = runExtract({ initialState });
|
|
238
|
+
const urls = result.media.filter((m) => m.type === 'image').map((m) => m.url);
|
|
239
|
+
expect(urls).toEqual(['https://sns-img-bd.xhscdn.com/keep.jpg']);
|
|
240
|
+
});
|
|
241
|
+
|
|
242
|
+
it('does not run DOM fallback when initial state yielded any image (preserves canonical order)', () => {
|
|
243
|
+
const initialState = {
|
|
244
|
+
note: {
|
|
245
|
+
noteDetailMap: {
|
|
246
|
+
'69f9716c000000003601f90e': {
|
|
247
|
+
note: {
|
|
248
|
+
imageList: [
|
|
249
|
+
{ urlDefault: 'https://sns-img-bd.xhscdn.com/canonical-only.jpg' },
|
|
250
|
+
],
|
|
251
|
+
},
|
|
252
|
+
},
|
|
253
|
+
},
|
|
254
|
+
},
|
|
255
|
+
};
|
|
256
|
+
const html = `
|
|
257
|
+
<div class="swiper-slide"><img src="https://sns-img-bd.xhscdn.com/dom-extra.jpg" /></div>
|
|
258
|
+
`;
|
|
259
|
+
const result = runExtract({ html, initialState });
|
|
260
|
+
const urls = result.media.filter((m) => m.type === 'image').map((m) => m.url);
|
|
261
|
+
expect(urls).toEqual(['https://sns-img-bd.xhscdn.com/canonical-only.jpg']);
|
|
262
|
+
});
|
|
263
|
+
|
|
264
|
+
it('uses only the current note entry from multi-note initial state maps', () => {
|
|
265
|
+
const initialState = {
|
|
266
|
+
note: {
|
|
267
|
+
noteDetailMap: {
|
|
268
|
+
othernote0000000000000001: {
|
|
269
|
+
note: {
|
|
270
|
+
imageList: [{ urlDefault: 'https://sns-img-bd.xhscdn.com/other.jpg' }],
|
|
271
|
+
video: { url: 'https://sns-video-bd.xhscdn.com/other.mp4' },
|
|
272
|
+
},
|
|
273
|
+
},
|
|
274
|
+
'69f9716c000000003601f90e': {
|
|
275
|
+
note: {
|
|
276
|
+
imageList: [
|
|
277
|
+
{ urlDefault: 'https://sns-img-bd.xhscdn.com/current-1.jpg' },
|
|
278
|
+
{ urlDefault: 'https://sns-img-bd.xhscdn.com/current-2.jpg' },
|
|
279
|
+
],
|
|
280
|
+
video: { url: 'https://sns-video-bd.xhscdn.com/current.mp4' },
|
|
281
|
+
},
|
|
282
|
+
},
|
|
283
|
+
},
|
|
284
|
+
},
|
|
285
|
+
};
|
|
286
|
+
const result = runExtract({ initialState });
|
|
287
|
+
const images = result.media.filter((m) => m.type === 'image').map((m) => m.url);
|
|
288
|
+
const videos = result.media.filter((m) => m.type === 'video').map((m) => m.url);
|
|
289
|
+
expect(images).toEqual([
|
|
290
|
+
'https://sns-img-bd.xhscdn.com/current-1.jpg',
|
|
291
|
+
'https://sns-img-bd.xhscdn.com/current-2.jpg',
|
|
292
|
+
]);
|
|
293
|
+
expect(videos).toEqual(['https://sns-video-bd.xhscdn.com/current.mp4']);
|
|
294
|
+
});
|
|
295
|
+
|
|
296
|
+
it('still resolves videos from __INITIAL_STATE__ alongside the image fix', () => {
|
|
297
|
+
const initialState = {
|
|
298
|
+
note: {
|
|
299
|
+
noteDetailMap: {
|
|
300
|
+
'69f9716c000000003601f90e': {
|
|
301
|
+
note: {
|
|
302
|
+
imageList: [{ urlDefault: 'https://sns-img-bd.xhscdn.com/cover.jpg' }],
|
|
303
|
+
video: { url: 'https://sns-video-bd.xhscdn.com/test.mp4' },
|
|
304
|
+
},
|
|
305
|
+
},
|
|
306
|
+
},
|
|
307
|
+
},
|
|
308
|
+
};
|
|
309
|
+
const result = runExtract({ initialState });
|
|
310
|
+
const images = result.media.filter((m) => m.type === 'image').map((m) => m.url);
|
|
311
|
+
const videos = result.media.filter((m) => m.type === 'video').map((m) => m.url);
|
|
312
|
+
expect(images).toEqual(['https://sns-img-bd.xhscdn.com/cover.jpg']);
|
|
313
|
+
expect(videos).toEqual(['https://sns-video-bd.xhscdn.com/test.mp4']);
|
|
314
|
+
expect(result.media.map((m) => m.type)).toEqual(['video', 'image']);
|
|
315
|
+
});
|
|
316
|
+
});
|
|
@@ -1,28 +1,9 @@
|
|
|
1
1
|
import { cli, Strategy } from '@jackwener/opencli/registry';
|
|
2
2
|
import { ArgumentError, AuthRequiredError, CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors';
|
|
3
|
-
|
|
4
|
-
function decodeEntity(codePoint) {
|
|
5
|
-
return Number.isInteger(codePoint) && codePoint >= 0 && codePoint <= 0x10FFFF
|
|
6
|
-
? String.fromCodePoint(codePoint)
|
|
7
|
-
: null;
|
|
8
|
-
}
|
|
3
|
+
import { stripHtml as stripHtmlText } from './text.js';
|
|
9
4
|
|
|
10
5
|
function stripHtml(html) {
|
|
11
|
-
|
|
12
|
-
return html
|
|
13
|
-
.replace(/<br\s*\/?\s*>/gi, '\n')
|
|
14
|
-
.replace(/<\/(?:p|div|h[1-6]|li|blockquote)>/gi, '\n\n')
|
|
15
|
-
.replace(/<[^>]+>/g, '')
|
|
16
|
-
.replace(/ /g, ' ')
|
|
17
|
-
.replace(/</g, '<')
|
|
18
|
-
.replace(/>/g, '>')
|
|
19
|
-
.replace(/&/g, '&')
|
|
20
|
-
.replace(/"/g, '"')
|
|
21
|
-
.replace(/'/g, "'")
|
|
22
|
-
.replace(/&#(\d+);/g, (entity, value) => decodeEntity(Number(value)) ?? entity)
|
|
23
|
-
.replace(/&#x([0-9a-f]+);/gi, (entity, value) => decodeEntity(Number.parseInt(value, 16)) ?? entity)
|
|
24
|
-
.replace(/\n{3,}/g, '\n\n')
|
|
25
|
-
.trim();
|
|
6
|
+
return stripHtmlText(html, { preserveBlocks: true });
|
|
26
7
|
}
|
|
27
8
|
|
|
28
9
|
const ANSWER_ID_RE = /^\d+$/;
|
|
@@ -1,38 +1,9 @@
|
|
|
1
1
|
import { cli, Strategy } from '@jackwener/opencli/registry';
|
|
2
2
|
import { ArgumentError, AuthRequiredError, CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors';
|
|
3
|
+
import { stripHtml as stripHtmlText } from './text.js';
|
|
3
4
|
|
|
4
|
-
// Light-weight HTML → text, preserving paragraph / heading / list-item
|
|
5
|
-
// line breaks. Zhihu answer `content` is HTML, so we map block-level
|
|
6
|
-
// closing tags + `<br>` to newlines before stripping the rest.
|
|
7
5
|
function stripHtml(html) {
|
|
8
|
-
|
|
9
|
-
return html
|
|
10
|
-
.replace(/<br\s*\/?\s*>/gi, '\n')
|
|
11
|
-
// Block-level closing tags become paragraph breaks (double
|
|
12
|
-
// newline) so the stripped text stays readable. The trailing
|
|
13
|
-
// `\n{3,}` collapse pass below normalizes accidental triples.
|
|
14
|
-
.replace(/<\/(?:p|div|h[1-6]|li|blockquote)>/gi, '\n\n')
|
|
15
|
-
.replace(/<[^>]+>/g, '')
|
|
16
|
-
.replace(/ /g, ' ')
|
|
17
|
-
.replace(/</g, '<')
|
|
18
|
-
.replace(/>/g, '>')
|
|
19
|
-
.replace(/&/g, '&')
|
|
20
|
-
.replace(/"/g, '"')
|
|
21
|
-
.replace(/'/g, "'")
|
|
22
|
-
.replace(/&#(\d+);/g, (_, value) => {
|
|
23
|
-
const codePoint = Number(value);
|
|
24
|
-
return Number.isInteger(codePoint) && codePoint >= 0 && codePoint <= 0x10FFFF
|
|
25
|
-
? String.fromCodePoint(codePoint)
|
|
26
|
-
: _;
|
|
27
|
-
})
|
|
28
|
-
.replace(/&#x([0-9a-f]+);/gi, (_, value) => {
|
|
29
|
-
const codePoint = Number.parseInt(value, 16);
|
|
30
|
-
return Number.isInteger(codePoint) && codePoint >= 0 && codePoint <= 0x10FFFF
|
|
31
|
-
? String.fromCodePoint(codePoint)
|
|
32
|
-
: _;
|
|
33
|
-
})
|
|
34
|
-
.replace(/\n{3,}/g, '\n\n')
|
|
35
|
-
.trim();
|
|
6
|
+
return stripHtmlText(html, { preserveBlocks: true });
|
|
36
7
|
}
|
|
37
8
|
|
|
38
9
|
const ANSWER_ID_RE = /^\d+$/;
|
package/clis/zhihu/collection.js
CHANGED
|
@@ -1,19 +1,7 @@
|
|
|
1
1
|
import { cli, Strategy } from '@jackwener/opencli/registry';
|
|
2
2
|
import { ArgumentError, AuthRequiredError, CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors';
|
|
3
3
|
import { log } from '@jackwener/opencli/logger';
|
|
4
|
-
|
|
5
|
-
function stripHtml(html) {
|
|
6
|
-
return html
|
|
7
|
-
.replace(/<[^>]+>/g, '')
|
|
8
|
-
.replace(/ /g, ' ')
|
|
9
|
-
.replace(/</g, '<')
|
|
10
|
-
.replace(/>/g, '>')
|
|
11
|
-
.replace(/&/g, '&')
|
|
12
|
-
.replace(/"/g, '"')
|
|
13
|
-
.replace(/<em>/g, '')
|
|
14
|
-
.replace(/<\/em>/g, '')
|
|
15
|
-
.trim();
|
|
16
|
-
}
|
|
4
|
+
import { stripHtml } from './text.js';
|
|
17
5
|
|
|
18
6
|
function validatePositiveInt(value, name) {
|
|
19
7
|
const n = Number(value);
|
|
@@ -106,7 +94,7 @@ function mapCollectionItem(item, rank) {
|
|
|
106
94
|
return {
|
|
107
95
|
rank,
|
|
108
96
|
type,
|
|
109
|
-
title: title.substring(0, 100),
|
|
97
|
+
title: stripHtml(title).substring(0, 100),
|
|
110
98
|
author,
|
|
111
99
|
votes,
|
|
112
100
|
excerpt,
|
|
@@ -34,10 +34,10 @@ describe('zhihu collection', () => {
|
|
|
34
34
|
content: {
|
|
35
35
|
type: 'answer',
|
|
36
36
|
id: 123456,
|
|
37
|
-
question: { id: 789012, title: 'Test Question' },
|
|
37
|
+
question: { id: 789012, title: '"Test" & Question' },
|
|
38
38
|
author: { name: 'test_author' },
|
|
39
39
|
voteup_count: 42,
|
|
40
|
-
content: '<p
|
|
40
|
+
content: '<p>"Test" & answer content</p>',
|
|
41
41
|
url: 'https://www.zhihu.com/question/789012/answer/123456',
|
|
42
42
|
},
|
|
43
43
|
},
|
|
@@ -54,9 +54,10 @@ describe('zhihu collection', () => {
|
|
|
54
54
|
expect(result[0]).toMatchObject({
|
|
55
55
|
rank: 1,
|
|
56
56
|
type: 'answer',
|
|
57
|
-
title: 'Test Question',
|
|
57
|
+
title: '"Test" & Question',
|
|
58
58
|
author: 'test_author',
|
|
59
59
|
votes: 42,
|
|
60
|
+
excerpt: '"Test" & answer content',
|
|
60
61
|
url: 'https://www.zhihu.com/question/789012/answer/123456',
|
|
61
62
|
});
|
|
62
63
|
|
package/clis/zhihu/question.js
CHANGED
|
@@ -1,14 +1,6 @@
|
|
|
1
1
|
import { cli, Strategy } from '@jackwener/opencli/registry';
|
|
2
2
|
import { AuthRequiredError, CliError } from '@jackwener/opencli/errors';
|
|
3
|
-
|
|
4
|
-
return html
|
|
5
|
-
.replace(/<[^>]+>/g, '')
|
|
6
|
-
.replace(/ /g, ' ')
|
|
7
|
-
.replace(/</g, '<')
|
|
8
|
-
.replace(/>/g, '>')
|
|
9
|
-
.replace(/&/g, '&')
|
|
10
|
-
.trim();
|
|
11
|
-
}
|
|
3
|
+
import { stripHtml } from './text.js';
|
|
12
4
|
|
|
13
5
|
function answerIdFromUrl(url) {
|
|
14
6
|
if (typeof url !== 'string') return '';
|
|
@@ -20,7 +20,7 @@ describe('zhihu question', () => {
|
|
|
20
20
|
id: '2036567240334653053',
|
|
21
21
|
author: { name: 'alice' },
|
|
22
22
|
voteup_count: 12,
|
|
23
|
-
content: 'Hello Zhihu',
|
|
23
|
+
content: '<p>"Hello" & Zhihu</p>',
|
|
24
24
|
},
|
|
25
25
|
],
|
|
26
26
|
};
|
|
@@ -33,7 +33,7 @@ describe('zhihu question', () => {
|
|
|
33
33
|
author: 'alice',
|
|
34
34
|
votes: 12,
|
|
35
35
|
url: 'https://www.zhihu.com/question/2021881398772981878/answer/2036567240334653053',
|
|
36
|
-
content: 'Hello Zhihu',
|
|
36
|
+
content: '"Hello" & Zhihu',
|
|
37
37
|
},
|
|
38
38
|
]);
|
|
39
39
|
expect(goto).toHaveBeenCalledWith('https://www.zhihu.com/question/2021881398772981878');
|
package/clis/zhihu/search.js
CHANGED
|
@@ -1,17 +1,6 @@
|
|
|
1
1
|
import { cli, Strategy } from '@jackwener/opencli/registry';
|
|
2
2
|
import { ArgumentError, AuthRequiredError, CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors';
|
|
3
|
-
|
|
4
|
-
function stripHtml(html) {
|
|
5
|
-
return (html || '')
|
|
6
|
-
.replace(/<[^>]+>/g, '')
|
|
7
|
-
.replace(/ /g, ' ')
|
|
8
|
-
.replace(/</g, '<')
|
|
9
|
-
.replace(/>/g, '>')
|
|
10
|
-
.replace(/&/g, '&')
|
|
11
|
-
.replace(/<em>/g, '')
|
|
12
|
-
.replace(/<\/em>/g, '')
|
|
13
|
-
.trim();
|
|
14
|
-
}
|
|
3
|
+
import { stripHtml } from './text.js';
|
|
15
4
|
|
|
16
5
|
function itemKey(item) {
|
|
17
6
|
const obj = item.object || {};
|
|
@@ -36,7 +36,7 @@ describe('zhihu search', () => {
|
|
|
36
36
|
type: 'answer',
|
|
37
37
|
author: { name: 'alice' },
|
|
38
38
|
voteup_count: 12,
|
|
39
|
-
question: { id: 'q1', name: '<em>Codex</em> question' },
|
|
39
|
+
question: { id: 'q1', name: '<em>Codex</em> "question"' },
|
|
40
40
|
},
|
|
41
41
|
},
|
|
42
42
|
{
|
|
@@ -57,7 +57,7 @@ describe('zhihu search', () => {
|
|
|
57
57
|
await expect(cmd.func(page, { query: 'codex', limit: 2 })).resolves.toEqual([
|
|
58
58
|
{
|
|
59
59
|
rank: 1,
|
|
60
|
-
title: 'Codex question',
|
|
60
|
+
title: 'Codex "question"',
|
|
61
61
|
type: 'answer',
|
|
62
62
|
author: 'alice',
|
|
63
63
|
votes: 12,
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
function decodeEntity(codePoint) {
|
|
2
|
+
return Number.isInteger(codePoint) && codePoint >= 0 && codePoint <= 0x10FFFF
|
|
3
|
+
? String.fromCodePoint(codePoint)
|
|
4
|
+
: null;
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
export function stripHtml(html, { preserveBlocks = false } = {}) {
|
|
8
|
+
if (!html) return '';
|
|
9
|
+
let text = String(html);
|
|
10
|
+
if (preserveBlocks) {
|
|
11
|
+
text = text
|
|
12
|
+
.replace(/<br\s*\/?\s*>/gi, '\n')
|
|
13
|
+
.replace(/<\/(?:p|div|h[1-6]|li|blockquote)>/gi, '\n\n');
|
|
14
|
+
}
|
|
15
|
+
return text
|
|
16
|
+
.replace(/<[^>]+>/g, '')
|
|
17
|
+
.replace(/ /g, ' ')
|
|
18
|
+
.replace(/</g, '<')
|
|
19
|
+
.replace(/>/g, '>')
|
|
20
|
+
.replace(/&/g, '&')
|
|
21
|
+
.replace(/"/g, '"')
|
|
22
|
+
.replace(/'/g, "'")
|
|
23
|
+
.replace(/&#(\d+);/g, (entity, value) => decodeEntity(Number(value)) ?? entity)
|
|
24
|
+
.replace(/&#x([0-9a-f]+);/gi, (entity, value) => decodeEntity(Number.parseInt(value, 16)) ?? entity)
|
|
25
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
26
|
+
.trim();
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export const __test__ = { decodeEntity };
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { stripHtml } from './text.js';
|
|
3
|
+
|
|
4
|
+
describe('zhihu text helpers', () => {
|
|
5
|
+
it('strips tags and decodes named entities in flat mode', () => {
|
|
6
|
+
expect(stripHtml('<em>Codex</em> & <CLI>')).toBe('Codex & <CLI>');
|
|
7
|
+
});
|
|
8
|
+
|
|
9
|
+
it('decodes decimal and hexadecimal numeric entities', () => {
|
|
10
|
+
expect(stripHtml('"中文" & 'test'')).toBe('"中文" & \'test\'');
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
it('keeps invalid numeric entities unchanged', () => {
|
|
14
|
+
expect(stripHtml('bad � entity')).toBe('bad � entity');
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
it('keeps list excerpts flat by default', () => {
|
|
18
|
+
expect(stripHtml('<p>first</p><br><p>second</p>')).toBe('firstsecond');
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
it('preserves block breaks when requested', () => {
|
|
22
|
+
expect(stripHtml('<p>first</p><br><p>second</p>', { preserveBlocks: true })).toBe('first\n\nsecond');
|
|
23
|
+
});
|
|
24
|
+
});
|
|
@@ -29,7 +29,19 @@ export function saveNetworkCache(session, entries, baseDir) {
|
|
|
29
29
|
savedAt: new Date().toISOString(),
|
|
30
30
|
entries,
|
|
31
31
|
};
|
|
32
|
-
|
|
32
|
+
// 0o600: entries can include auth tokens and PII from captured response
|
|
33
|
+
// bodies. fchmod before writing also tightens a pre-existing broad file.
|
|
34
|
+
let fd;
|
|
35
|
+
try {
|
|
36
|
+
fd = fs.openSync(target, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_TRUNC, 0o600);
|
|
37
|
+
fs.fchmodSync(fd, 0o600);
|
|
38
|
+
fs.writeFileSync(fd, JSON.stringify(payload), 'utf8');
|
|
39
|
+
}
|
|
40
|
+
finally {
|
|
41
|
+
if (fd !== undefined) {
|
|
42
|
+
fs.closeSync(fd);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
33
45
|
}
|
|
34
46
|
export function loadNetworkCache(session, opts = {}) {
|
|
35
47
|
const target = getCachePath(session, opts.baseDir);
|
|
@@ -55,4 +55,21 @@ describe('network-cache', () => {
|
|
|
55
55
|
expect(findEntry(file, 'B')?.key).toBe('B');
|
|
56
56
|
expect(findEntry(file, 'missing')).toBeNull();
|
|
57
57
|
});
|
|
58
|
+
it.skipIf(process.platform === 'win32')('writes the cache file with 0o600 owner-only permissions', () => {
|
|
59
|
+
saveNetworkCache('ws', [makeEntry('UserTweets')], baseDir);
|
|
60
|
+
const target = getCachePath('ws', baseDir);
|
|
61
|
+
const mode = fs.statSync(target).mode & 0o777;
|
|
62
|
+
expect(mode).toBe(0o600);
|
|
63
|
+
});
|
|
64
|
+
it.skipIf(process.platform === 'win32')('tightens an existing cache file before rewriting it', () => {
|
|
65
|
+
const target = getCachePath('ws', baseDir);
|
|
66
|
+
fs.mkdirSync(path.dirname(target), { recursive: true });
|
|
67
|
+
fs.writeFileSync(target, '{"version":1,"session":"ws","savedAt":"old","entries":[]}', { mode: 0o644 });
|
|
68
|
+
saveNetworkCache('ws', [makeEntry('UserTweets')], baseDir);
|
|
69
|
+
const mode = fs.statSync(target).mode & 0o777;
|
|
70
|
+
expect(mode).toBe(0o600);
|
|
71
|
+
const reloaded = loadNetworkCache('ws', { baseDir });
|
|
72
|
+
expect(reloaded.status).toBe('ok');
|
|
73
|
+
expect(reloaded.file?.entries[0].key).toBe('UserTweets');
|
|
74
|
+
});
|
|
58
75
|
});
|
|
@@ -165,7 +165,19 @@ export function exportCookiesToNetscape(cookies, filePath) {
|
|
|
165
165
|
lines.push(`${domain}\t${includeSubdomains}\t${cookiePath}\t${secure}\t${expiry}\t${safeName}\t${safeValue}`);
|
|
166
166
|
}
|
|
167
167
|
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
|
168
|
-
|
|
168
|
+
// 0o600: file holds live session cookies, must be owner-only. fchmod before
|
|
169
|
+
// writing also tightens a pre-existing broad file before new secrets land.
|
|
170
|
+
let fd;
|
|
171
|
+
try {
|
|
172
|
+
fd = fs.openSync(filePath, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_TRUNC, 0o600);
|
|
173
|
+
fs.fchmodSync(fd, 0o600);
|
|
174
|
+
fs.writeFileSync(fd, lines.join('\n'), 'utf8');
|
|
175
|
+
}
|
|
176
|
+
finally {
|
|
177
|
+
if (fd !== undefined) {
|
|
178
|
+
fs.closeSync(fd);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
169
181
|
}
|
|
170
182
|
export function formatCookieHeader(cookies) {
|
|
171
183
|
return cookies.map((cookie) => `${cookie.name}=${cookie.value}`).join('; ');
|
|
@@ -3,7 +3,7 @@ import * as http from 'node:http';
|
|
|
3
3
|
import * as os from 'node:os';
|
|
4
4
|
import * as path from 'node:path';
|
|
5
5
|
import { afterEach, describe, expect, it, vi } from 'vitest';
|
|
6
|
-
import { formatCookieHeader, httpDownload, resolveRedirectUrl } from './index.js';
|
|
6
|
+
import { exportCookiesToNetscape, formatCookieHeader, httpDownload, resolveRedirectUrl } from './index.js';
|
|
7
7
|
const servers = [];
|
|
8
8
|
const tempDirs = [];
|
|
9
9
|
afterEach(async () => {
|
|
@@ -115,4 +115,26 @@ describe('download helpers', { retry: process.platform === 'win32' ? 2 : 0 }, ()
|
|
|
115
115
|
expect(result).toEqual({ success: true, size: 2 });
|
|
116
116
|
expect(fs.readFileSync(destPath, 'utf8')).toBe('ok');
|
|
117
117
|
});
|
|
118
|
+
it.skipIf(process.platform === 'win32')('writes the Netscape cookie file with 0o600 owner-only permissions', async () => {
|
|
119
|
+
const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'opencli-dl-'));
|
|
120
|
+
tempDirs.push(tempDir);
|
|
121
|
+
const cookiesPath = path.join(tempDir, 'cookies.txt');
|
|
122
|
+
exportCookiesToNetscape([
|
|
123
|
+
{ name: 'sid', value: 'secret', domain: 'example.com' },
|
|
124
|
+
], cookiesPath);
|
|
125
|
+
const mode = fs.statSync(cookiesPath).mode & 0o777;
|
|
126
|
+
expect(mode).toBe(0o600);
|
|
127
|
+
});
|
|
128
|
+
it.skipIf(process.platform === 'win32')('tightens an existing Netscape cookie file before rewriting it', async () => {
|
|
129
|
+
const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'opencli-dl-'));
|
|
130
|
+
tempDirs.push(tempDir);
|
|
131
|
+
const cookiesPath = path.join(tempDir, 'cookies.txt');
|
|
132
|
+
fs.writeFileSync(cookiesPath, 'stale-cookie', { mode: 0o644 });
|
|
133
|
+
exportCookiesToNetscape([
|
|
134
|
+
{ name: 'sid', value: 'secret', domain: 'example.com' },
|
|
135
|
+
], cookiesPath);
|
|
136
|
+
const mode = fs.statSync(cookiesPath).mode & 0o777;
|
|
137
|
+
expect(mode).toBe(0o600);
|
|
138
|
+
expect(fs.readFileSync(cookiesPath, 'utf8')).toContain('sid\tsecret');
|
|
139
|
+
});
|
|
118
140
|
});
|
|
@@ -29,7 +29,9 @@ async function startServer(handler) {
|
|
|
29
29
|
}
|
|
30
30
|
return `http://127.0.0.1:${address.port}`;
|
|
31
31
|
}
|
|
32
|
-
|
|
32
|
+
// Windows runners occasionally exceed the default 5s timeout on the first
|
|
33
|
+
// http.createServer + downloadMedia roundtrip (cold-start cost on a loaded VM).
|
|
34
|
+
describe('media downloads', { retry: process.platform === 'win32' ? 2 : 0 }, () => {
|
|
33
35
|
it('keeps custom filenames inside the output directory', async () => {
|
|
34
36
|
const baseUrl = await startServer((_req, res) => {
|
|
35
37
|
res.statusCode = 200;
|
|
@@ -5,11 +5,11 @@
|
|
|
5
5
|
* Format bytes as human-readable string (KB, MB, GB).
|
|
6
6
|
*/
|
|
7
7
|
export function formatBytes(bytes) {
|
|
8
|
-
if (bytes
|
|
8
|
+
if (!Number.isFinite(bytes) || bytes < 1)
|
|
9
9
|
return '0 B';
|
|
10
10
|
const k = 1024;
|
|
11
11
|
const sizes = ['B', 'KB', 'MB', 'GB', 'TB'];
|
|
12
|
-
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
|
12
|
+
const i = Math.min(sizes.length - 1, Math.floor(Math.log(bytes) / Math.log(k)));
|
|
13
13
|
return `${(bytes / Math.pow(k, i)).toFixed(1)} ${sizes[i]}`;
|
|
14
14
|
}
|
|
15
15
|
/**
|
|
@@ -1,5 +1,16 @@
|
|
|
1
1
|
import { afterEach, describe, expect, it, vi } from 'vitest';
|
|
2
|
-
import { createProgressBar } from './progress.js';
|
|
2
|
+
import { createProgressBar, formatBytes } from './progress.js';
|
|
3
|
+
describe('formatBytes', () => {
|
|
4
|
+
it('returns a stable zero-byte label for invalid or sub-byte values', () => {
|
|
5
|
+
expect(formatBytes(-1)).toBe('0 B');
|
|
6
|
+
expect(formatBytes(0.5)).toBe('0 B');
|
|
7
|
+
expect(formatBytes(Number.NaN)).toBe('0 B');
|
|
8
|
+
expect(formatBytes(Number.POSITIVE_INFINITY)).toBe('0 B');
|
|
9
|
+
});
|
|
10
|
+
it('caps very large values at the largest supported unit', () => {
|
|
11
|
+
expect(formatBytes(1024 ** 5)).toBe('1024.0 TB');
|
|
12
|
+
});
|
|
13
|
+
});
|
|
3
14
|
describe('download progress display', () => {
|
|
4
15
|
afterEach(() => {
|
|
5
16
|
vi.restoreAllMocks();
|
package/dist/src/output.js
CHANGED
|
@@ -94,7 +94,7 @@ function renderPlain(data, opts) {
|
|
|
94
94
|
const entries = Object.entries(row);
|
|
95
95
|
if (entries.length === 1) {
|
|
96
96
|
const [key, value] = entries[0];
|
|
97
|
-
if (key === 'response' || key === 'content' || key === 'text' || key === 'value') {
|
|
97
|
+
if (key === 'response' || key === 'content' || key === 'markdown' || key === 'text' || key === 'value') {
|
|
98
98
|
console.log(String(value ?? ''));
|
|
99
99
|
return;
|
|
100
100
|
}
|
|
@@ -113,6 +113,16 @@ function renderMarkdown(data, opts) {
|
|
|
113
113
|
const rows = normalizeRows(data);
|
|
114
114
|
if (!rows.length)
|
|
115
115
|
return;
|
|
116
|
+
if (rows.length === 1) {
|
|
117
|
+
const entries = Object.entries(rows[0]);
|
|
118
|
+
if (entries.length === 1) {
|
|
119
|
+
const [key, value] = entries[0];
|
|
120
|
+
if (key === 'content' || key === 'markdown' || key === 'text' || key === 'value') {
|
|
121
|
+
console.log(String(value ?? ''));
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
116
126
|
const columns = resolveColumns(rows, opts);
|
|
117
127
|
console.log('| ' + columns.join(' | ') + ' |');
|
|
118
128
|
console.log('| ' + columns.map(() => '---').join(' | ') + ' |');
|