@crawlee/cli 3.13.5 → 4.0.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commands/CreateProjectCommand.d.ts.map +1 -1
- package/commands/CreateProjectCommand.js +56 -88
- package/commands/CreateProjectCommand.js.map +1 -1
- package/commands/InstallPlaywrightBrowsersCommand.js +26 -53
- package/commands/InstallPlaywrightBrowsersCommand.js.map +1 -1
- package/commands/RunProjectCommand.js +17 -38
- package/commands/RunProjectCommand.js.map +1 -1
- package/index.js +11 -21
- package/index.js.map +1 -1
- package/package.json +10 -17
- package/tsconfig.build.tsbuildinfo +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CreateProjectCommand.d.ts","sourceRoot":"","sources":["../../src/commands/CreateProjectCommand.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,kBAAkB,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAErE,UAAU,iBAAiB;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAgGD,qBAAa,oBAAoB,CAAC,CAAC,CAAE,YAAW,aAAa,CAAC,CAAC,EAAE,iBAAiB,CAAC;IAC/E,OAAO,SAA2B;IAClC,QAAQ,SAAmF;IAC3F,OAAO,GAAU,MAAM,IAAI,CAAC,CAAC,CAAC,sCAa5B;IAEF;;OAEG;IACG,OAAO,CAAC,IAAI,EAAE,kBAAkB,CAAC,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"CreateProjectCommand.d.ts","sourceRoot":"","sources":["../../src/commands/CreateProjectCommand.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,kBAAkB,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAErE,UAAU,iBAAiB;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAgGD,qBAAa,oBAAoB,CAAC,CAAC,CAAE,YAAW,aAAa,CAAC,CAAC,EAAE,iBAAiB,CAAC;IAC/E,OAAO,SAA2B;IAClC,QAAQ,SAAmF;IAC3F,OAAO,GAAU,MAAM,IAAI,CAAC,CAAC,CAAC,sCAa5B;IAEF;;OAEG;IACG,OAAO,CAAC,IAAI,EAAE,kBAAkB,CAAC,iBAAiB,CAAC;CA8D5D"}
|
|
@@ -1,17 +1,13 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
const templates_1 = require("@crawlee/templates");
|
|
12
|
-
const ansi_colors_1 = tslib_1.__importDefault(require("ansi-colors"));
|
|
13
|
-
const fs_extra_1 = require("fs-extra");
|
|
14
|
-
const inquirer_1 = require("inquirer");
|
|
1
|
+
import { execSync } from 'node:child_process';
|
|
2
|
+
import { mkdirSync } from 'node:fs';
|
|
3
|
+
import { readFile, writeFile } from 'node:fs/promises';
|
|
4
|
+
import { get } from 'node:https';
|
|
5
|
+
import { dirname, join, resolve } from 'node:path';
|
|
6
|
+
import { setTimeout } from 'node:timers/promises';
|
|
7
|
+
import { fetchManifest } from '@crawlee/templates';
|
|
8
|
+
import { input, select } from '@inquirer/prompts';
|
|
9
|
+
import colors from 'ansi-colors';
|
|
10
|
+
import { ensureDir } from 'fs-extra/esm';
|
|
15
11
|
function validateProjectName(name) {
|
|
16
12
|
if (name.length === 0) {
|
|
17
13
|
throw new Error('The project name cannot be empty string.');
|
|
@@ -19,9 +15,9 @@ function validateProjectName(name) {
|
|
|
19
15
|
}
|
|
20
16
|
async function rewrite(path, replacer) {
|
|
21
17
|
try {
|
|
22
|
-
const file = await
|
|
18
|
+
const file = await readFile(path, 'utf8');
|
|
23
19
|
const replaced = replacer(file);
|
|
24
|
-
await
|
|
20
|
+
await writeFile(path, replaced);
|
|
25
21
|
}
|
|
26
22
|
catch {
|
|
27
23
|
// not found
|
|
@@ -38,24 +34,24 @@ async function withRetries(func, retries, label) {
|
|
|
38
34
|
attempt++;
|
|
39
35
|
lastError = error;
|
|
40
36
|
if (attempt < retries) {
|
|
41
|
-
console.warn(`${
|
|
37
|
+
console.warn(`${colors.yellow(`[${label}]`)}: Attempt ${attempt + 1} of ${retries} failed, and will be retried`, error.message || error);
|
|
42
38
|
}
|
|
43
39
|
// Wait 2500ms + (2500 * retries) before giving up to give it some time between retries
|
|
44
|
-
await
|
|
40
|
+
await setTimeout(2500 + 2500 * attempt);
|
|
45
41
|
}
|
|
46
42
|
}
|
|
47
|
-
throw new Error(`${
|
|
43
|
+
throw new Error(`${colors.red(`[${label}]`)}: All ${retries} attempts failed, and will not be retried\n\n${lastError.stack || lastError}`);
|
|
48
44
|
}
|
|
49
45
|
async function downloadTemplateFilesToDisk(template, destinationDirectory) {
|
|
50
46
|
const promises = [];
|
|
51
47
|
for (const file of template.files) {
|
|
52
48
|
const promise = async () => downloadFile(file.url).then(async (buffer) => {
|
|
53
49
|
// Make sure the folder for the file exists
|
|
54
|
-
const fileDirName =
|
|
55
|
-
const fileFolder =
|
|
56
|
-
await
|
|
50
|
+
const fileDirName = dirname(file.path);
|
|
51
|
+
const fileFolder = resolve(destinationDirectory, fileDirName);
|
|
52
|
+
await ensureDir(fileFolder);
|
|
57
53
|
// Write the actual file
|
|
58
|
-
await
|
|
54
|
+
await writeFile(resolve(destinationDirectory, file.path), buffer);
|
|
59
55
|
});
|
|
60
56
|
promises.push(withRetries(promise, 3, `Template: ${template.name}, file: ${file.path}`));
|
|
61
57
|
}
|
|
@@ -63,7 +59,7 @@ async function downloadTemplateFilesToDisk(template, destinationDirectory) {
|
|
|
63
59
|
}
|
|
64
60
|
async function downloadFile(url) {
|
|
65
61
|
return new Promise((promiseResolve, reject) => {
|
|
66
|
-
|
|
62
|
+
get(url, async (res) => {
|
|
67
63
|
const bytes = [];
|
|
68
64
|
res.on('error', (err) => reject(err));
|
|
69
65
|
for await (const byte of res) {
|
|
@@ -78,39 +74,22 @@ async function downloadFile(url) {
|
|
|
78
74
|
}).on('error', (err) => reject(err));
|
|
79
75
|
});
|
|
80
76
|
}
|
|
81
|
-
class CreateProjectCommand {
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
77
|
+
export class CreateProjectCommand {
|
|
78
|
+
command = 'create [project-name]';
|
|
79
|
+
describe = 'Creates a new Crawlee project directory from a selected boilerplate template.';
|
|
80
|
+
builder = async (args) => {
|
|
81
|
+
const manifest = await fetchManifest();
|
|
82
|
+
const choices = manifest.templates.map((t) => t.name);
|
|
83
|
+
args.positional('project-name', {
|
|
84
|
+
describe: 'Name of the new project folder.',
|
|
88
85
|
});
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
value: 'Creates a new Crawlee project directory from a selected boilerplate template.'
|
|
86
|
+
args.option('template', {
|
|
87
|
+
alias: 't',
|
|
88
|
+
choices,
|
|
89
|
+
describe: 'Template for the project. If not provided, the command will prompt for it.',
|
|
94
90
|
});
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
configurable: true,
|
|
98
|
-
writable: true,
|
|
99
|
-
value: async (args) => {
|
|
100
|
-
const manifest = await (0, templates_1.fetchManifest)();
|
|
101
|
-
const choices = manifest.templates.map((t) => t.name);
|
|
102
|
-
args.positional('project-name', {
|
|
103
|
-
describe: 'Name of the new project folder.',
|
|
104
|
-
});
|
|
105
|
-
args.option('template', {
|
|
106
|
-
alias: 't',
|
|
107
|
-
choices,
|
|
108
|
-
describe: 'Template for the project. If not provided, the command will prompt for it.',
|
|
109
|
-
});
|
|
110
|
-
return args;
|
|
111
|
-
}
|
|
112
|
-
});
|
|
113
|
-
}
|
|
91
|
+
return args;
|
|
92
|
+
};
|
|
114
93
|
/**
|
|
115
94
|
* @inheritDoc
|
|
116
95
|
*/
|
|
@@ -118,48 +97,38 @@ class CreateProjectCommand {
|
|
|
118
97
|
let { projectName, template } = args;
|
|
119
98
|
// Check proper format of projectName
|
|
120
99
|
if (!projectName) {
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
return err.message;
|
|
132
|
-
}
|
|
133
|
-
return true;
|
|
134
|
-
},
|
|
100
|
+
projectName = await input({
|
|
101
|
+
message: 'Name of the new project folder:',
|
|
102
|
+
validate: (promptText) => {
|
|
103
|
+
try {
|
|
104
|
+
validateProjectName(promptText);
|
|
105
|
+
}
|
|
106
|
+
catch (err) {
|
|
107
|
+
return err.message;
|
|
108
|
+
}
|
|
109
|
+
return true;
|
|
135
110
|
},
|
|
136
|
-
|
|
137
|
-
({ projectName } = projectNamePrompt);
|
|
111
|
+
});
|
|
138
112
|
}
|
|
139
113
|
else {
|
|
140
114
|
validateProjectName(projectName);
|
|
141
115
|
}
|
|
142
|
-
const manifest = await withRetries(
|
|
116
|
+
const manifest = await withRetries(fetchManifest, 5, 'Template Manifest');
|
|
143
117
|
const choices = manifest.templates.map((t) => ({
|
|
144
118
|
value: t.name,
|
|
145
119
|
name: t.description,
|
|
146
120
|
}));
|
|
147
121
|
if (!template) {
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
default: choices[0],
|
|
154
|
-
choices,
|
|
155
|
-
},
|
|
156
|
-
]);
|
|
157
|
-
template = answer.template;
|
|
122
|
+
template = await select({
|
|
123
|
+
message: 'Please select the template for your new Crawlee project',
|
|
124
|
+
default: choices[0],
|
|
125
|
+
choices,
|
|
126
|
+
});
|
|
158
127
|
}
|
|
159
|
-
const projectDir =
|
|
128
|
+
const projectDir = join(process.cwd(), projectName);
|
|
160
129
|
// Create project directory structure
|
|
161
130
|
try {
|
|
162
|
-
|
|
131
|
+
mkdirSync(projectDir);
|
|
163
132
|
}
|
|
164
133
|
catch (err) {
|
|
165
134
|
if (err.code && err.code === 'EEXIST') {
|
|
@@ -170,12 +139,11 @@ class CreateProjectCommand {
|
|
|
170
139
|
}
|
|
171
140
|
const templateData = manifest.templates.find((item) => item.name === template);
|
|
172
141
|
await downloadTemplateFilesToDisk(templateData, projectDir);
|
|
173
|
-
await rewrite(
|
|
142
|
+
await rewrite(resolve(projectDir, 'package.json'), (pkg) => pkg.replace(/"name": "[\w-]+"/, `"name": "${projectName}"`));
|
|
174
143
|
// Run npm install in project dir.
|
|
175
144
|
const npm = /^win/.test(process.platform) ? 'npm.cmd' : 'npm';
|
|
176
|
-
|
|
177
|
-
console.log(
|
|
145
|
+
execSync(`${npm} install`, { cwd: projectDir, stdio: 'inherit' });
|
|
146
|
+
console.log(colors.green(`Project ${projectName} was created. To run it, run "cd ${projectName}" and "npm start".`));
|
|
178
147
|
}
|
|
179
148
|
}
|
|
180
|
-
exports.CreateProjectCommand = CreateProjectCommand;
|
|
181
149
|
//# sourceMappingURL=CreateProjectCommand.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CreateProjectCommand.js","sourceRoot":"","sources":["../../src/commands/CreateProjectCommand.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"CreateProjectCommand.js","sourceRoot":"","sources":["../../src/commands/CreateProjectCommand.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,GAAG,EAAE,MAAM,YAAY,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAGlD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAQzC,SAAS,mBAAmB,CAAC,IAAY;IACrC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAChE,CAAC;AACL,CAAC;AAED,KAAK,UAAU,OAAO,CAAC,IAAY,EAAE,QAAkC;IACnE,IAAI,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC1C,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QAChC,MAAM,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACL,YAAY;IAChB,CAAC;AACL,CAAC;AAED,KAAK,UAAU,WAAW,CACtB,IAAO,EACP,OAAe,EACf,KAAa;IAEb,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,SAAc,CAAC;IAEnB,OAAO,OAAO,GAAG,OAAO,EAAE,CAAC;QACvB,IAAI,CAAC;YACD,OAAO,CAAC,MAAM,IAAI,EAAE,CAA2B,CAAC;QACpD,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YAClB,OAAO,EAAE,CAAC;YACV,SAAS,GAAG,KAAK,CAAC;YAElB,IAAI,OAAO,GAAG,OAAO,EAAE,CAAC;gBACpB,OAAO,CAAC,IAAI,CACR,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,GAAG,CAAC,aAAa,OAAO,GAAG,CAAC,OAAO,OAAO,8BAA8B,EAClG,KAAK,CAAC,OAAO,IAAI,KAAK,CACzB,CAAC;YACN,CAAC;YAED,uFAAuF;YACvF,MAAM,UAAU,CAAC,IAAI,GAAG,IAAI,GAAG,OAAO,CAAC,CAAC;QAC5C,CAAC;IACL,CAAC;IAED,MAAM,IAAI,KAAK,CACX,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,SAAS,OAAO,gDACvC,SAAS,CAAC,KAAK,IAAI,SACvB,EAAE,CACL,CAAC;AACN,CAAC;AAED,KAAK,UAAU,2BAA2B,CAAC,QAAkB,EAAE,oBAA4B;IACvF,MAAM,QAAQ,GAAoB,EAAE,CAAC;IAErC,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,KAAK,IAAI,EAAE,CACvB,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;YACzC,2CAA2C;YAC3C,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACvC,MAAM,UAAU,GAAG,OAAO,CAAC,oBAAoB,EAAE,WAAW,CAAC,CAAC;YAC9D,MAAM,SAAS,CAAC,UAAU,CAAC,CAAC;YAE5B,wBAAwB;YACxB,MAAM,SAAS,CAAC,OAAO,CAAC,oBAAoB,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;QAEP,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,EAAE,aAAa,QAAQ,CAAC,IAAI,WAAW,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC7F,CAAC;IAED,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;AAChC,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,GAAW;IACnC,OAAO,IAAI,OAAO,CAAS,CAAC,cAAc,EAAE,MAAM,EAAE,EAAE;QAClD,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;YACnB,MAAM,KAAK,GAAa,EAAE,CAAC;YAE3B,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAEtC,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,GAAG,EAAE,CAAC;gBAC3B,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrB,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAElC,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;gBACzB,MAAM,CAAC,IAAI,KAAK,CAAC,YAAY,GAAG,CAAC,UAAU,IAAI,GAAG,CAAC,aAAa,KAAK,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC/F,OAAO;YACX,CAAC;YAED,cAAc,CAAC,IAAI,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;AACP,CAAC;AAED,MAAM,OAAO,oBAAoB;IAC7B,OAAO,GAAG,uBAAuB,CAAC;IAClC,QAAQ,GAAG,+EAA+E,CAAC;IAC3F,OAAO,GAAG,KAAK,EAAE,IAAa,EAAE,EAAE;QAC9B,MAAM,QAAQ,GAAG,MAAM,aAAa,EAAE,CAAC;QACvC,MAAM,OAAO,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAEtD,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE;YAC5B,QAAQ,EAAE,iCAAiC;SAC9C,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE;YACpB,KAAK,EAAE,GAAG;YACV,OAAO;YACP,QAAQ,EAAE,4EAA4E;SACzF,CAAC,CAAC;QACH,OAAO,IAA+B,CAAC;IAC3C,CAAC,CAAC;IAEF;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,IAA2C;QACrD,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;QAErC,qCAAqC;QACrC,IAAI,CAAC,WAAW,EAAE,CAAC;YACf,WAAW,GAAG,MAAM,KAAK,CAAC;gBACtB,OAAO,EAAE,iCAAiC;gBAC1C,QAAQ,EAAE,CAAC,UAAU,EAAE,EAAE;oBACrB,IAAI,CAAC;wBACD,mBAAmB,CAAC,UAAU,CAAC,CAAC;oBACpC,CAAC;oBAAC,OAAO,GAAQ,EAAE,CAAC;wBAChB,OAAO,GAAG,CAAC,OAAO,CAAC;oBACvB,CAAC;oBACD,OAAO,IAAI,CAAC;gBAChB,CAAC;aACJ,CAAC,CAAC;QACP,CAAC;aAAM,CAAC;YACJ,mBAAmB,CAAC,WAAW,CAAC,CAAC;QACrC,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,aAAa,EAAE,CAAC,EAAE,mBAAmB,CAAC,CAAC;QAC1E,MAAM,OAAO,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3C,KAAK,EAAE,CAAC,CAAC,IAAI;YACb,IAAI,EAAE,CAAC,CAAC,WAAW;SACtB,CAAC,CAAC,CAAC;QAEJ,IAAI,CAAC,QAAQ,EAAE,CAAC;YACZ,QAAQ,GAAG,MAAM,MAAM,CAAC;gBACpB,OAAO,EAAE,yDAAyD;gBAClE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;gBACnB,OAAO;aACV,CAAC,CAAC;QACP,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,WAAY,CAAC,CAAC;QAErD,qCAAqC;QACrC,IAAI,CAAC;YACD,SAAS,CAAC,UAAU,CAAC,CAAC;QAC1B,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAChB,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACpC,OAAO,CAAC,KAAK,CAAC,iDAAiD,WAAW,mBAAmB,CAAC,CAAC;gBAC/F,OAAO;YACX,CAAC;YACD,MAAM,GAAG,CAAC;QACd,CAAC;QAED,MAAM,YAAY,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAE,CAAC;QAEhF,MAAM,2BAA2B,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;QAC5D,MAAM,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,cAAc,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CACvD,GAAG,CAAC,OAAO,CAAC,kBAAkB,EAAE,YAAY,WAAW,GAAG,CAAC,CAC9D,CAAC;QAEF,kCAAkC;QAClC,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC;QAC9D,QAAQ,CAAC,GAAG,GAAG,UAAU,EAAE,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;QAElE,OAAO,CAAC,GAAG,CACP,MAAM,CAAC,KAAK,CAAC,WAAW,WAAW,oCAAoC,WAAW,oBAAoB,CAAC,CAC1G,CAAC;IACN,CAAC;CACJ"}
|
|
@@ -1,58 +1,31 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.InstallPlaywrightBrowsersCommand = void 0;
|
|
4
|
-
const tslib_1 = require("tslib");
|
|
5
|
-
const node_child_process_1 = require("node:child_process");
|
|
6
|
-
const ansi_colors_1 = tslib_1.__importDefault(require("ansi-colors"));
|
|
1
|
+
import { execSync } from 'node:child_process';
|
|
2
|
+
import ansiColors from 'ansi-colors';
|
|
7
3
|
const envVariable = 'CRAWLEE_SKIP_BROWSER_INSTALL';
|
|
8
|
-
class InstallPlaywrightBrowsersCommand {
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
4
|
+
export class InstallPlaywrightBrowsersCommand {
|
|
5
|
+
command = 'install-playwright-browsers';
|
|
6
|
+
describe = 'Installs browsers needed by Playwright for local testing';
|
|
7
|
+
builder = async (args) => {
|
|
8
|
+
args.options('force', {
|
|
9
|
+
alias: 'f',
|
|
10
|
+
default: false,
|
|
11
|
+
type: 'boolean',
|
|
12
|
+
describe: 'Use `--force` to force installation of browsers even if the environment is marked as having them.',
|
|
15
13
|
});
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
enumerable: true,
|
|
24
|
-
configurable: true,
|
|
25
|
-
writable: true,
|
|
26
|
-
value: async (args) => {
|
|
27
|
-
args.options('force', {
|
|
28
|
-
alias: 'f',
|
|
29
|
-
default: false,
|
|
30
|
-
type: 'boolean',
|
|
31
|
-
describe: 'Use `--force` to force installation of browsers even if the environment is marked as having them.',
|
|
32
|
-
});
|
|
33
|
-
return args;
|
|
34
|
-
}
|
|
35
|
-
});
|
|
36
|
-
Object.defineProperty(this, "handler", {
|
|
37
|
-
enumerable: true,
|
|
38
|
-
configurable: true,
|
|
39
|
-
writable: true,
|
|
40
|
-
value: (args) => {
|
|
41
|
-
if (process.env[envVariable]) {
|
|
42
|
-
if (!args.force) {
|
|
43
|
-
console.log(ansi_colors_1.default.green('Browsers are already installed!'));
|
|
44
|
-
return;
|
|
45
|
-
}
|
|
46
|
-
console.warn(ansi_colors_1.default.yellow('Installing Playwright browsers in an environment where browsers have already been installed...'));
|
|
47
|
-
}
|
|
48
|
-
else {
|
|
49
|
-
console.log(ansi_colors_1.default.green('Installing Playwright browsers...'));
|
|
50
|
-
}
|
|
51
|
-
// TODO: detect package manager
|
|
52
|
-
(0, node_child_process_1.execSync)(`npx playwright install`, { stdio: 'inherit' });
|
|
14
|
+
return args;
|
|
15
|
+
};
|
|
16
|
+
handler = (args) => {
|
|
17
|
+
if (process.env[envVariable]) {
|
|
18
|
+
if (!args.force) {
|
|
19
|
+
console.log(ansiColors.green('Browsers are already installed!'));
|
|
20
|
+
return;
|
|
53
21
|
}
|
|
54
|
-
|
|
55
|
-
|
|
22
|
+
console.warn(ansiColors.yellow('Installing Playwright browsers in an environment where browsers have already been installed...'));
|
|
23
|
+
}
|
|
24
|
+
else {
|
|
25
|
+
console.log(ansiColors.green('Installing Playwright browsers...'));
|
|
26
|
+
}
|
|
27
|
+
// TODO: detect package manager
|
|
28
|
+
execSync(`npx playwright install`, { stdio: 'inherit' });
|
|
29
|
+
};
|
|
56
30
|
}
|
|
57
|
-
exports.InstallPlaywrightBrowsersCommand = InstallPlaywrightBrowsersCommand;
|
|
58
31
|
//# sourceMappingURL=InstallPlaywrightBrowsersCommand.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"InstallPlaywrightBrowsersCommand.js","sourceRoot":"","sources":["../../src/commands/InstallPlaywrightBrowsersCommand.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"InstallPlaywrightBrowsersCommand.js","sourceRoot":"","sources":["../../src/commands/InstallPlaywrightBrowsersCommand.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAE9C,OAAO,UAAU,MAAM,aAAa,CAAC;AAGrC,MAAM,WAAW,GAAG,8BAA8B,CAAC;AAMnD,MAAM,OAAO,gCAAgC;IACzC,OAAO,GAAG,6BAA6B,CAAC;IACxC,QAAQ,GAAG,0DAA0D,CAAC;IAEtE,OAAO,GAAG,KAAK,EAAE,IAAa,EAAE,EAAE;QAC9B,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE;YAClB,KAAK,EAAE,GAAG;YACV,OAAO,EAAE,KAAK;YACd,IAAI,EAAE,SAAS;YACf,QAAQ,EACJ,mGAAmG;SAC1G,CAAC,CAAC;QAEH,OAAO,IAA2C,CAAC;IACvD,CAAC,CAAC;IAEF,OAAO,GAAG,CAAC,IAAuD,EAAE,EAAE;QAClE,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;YAC3B,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACd,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC,CAAC;gBACjE,OAAO;YACX,CAAC;YAED,OAAO,CAAC,IAAI,CACR,UAAU,CAAC,MAAM,CACb,gGAAgG,CACnG,CACJ,CAAC;QACN,CAAC;aAAM,CAAC;YACJ,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC,CAAC;QACvE,CAAC;QAED,+BAA+B;QAC/B,QAAQ,CAAC,wBAAwB,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;IAC7D,CAAC,CAAC;CACL"}
|
|
@@ -1,41 +1,21 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
value: 'run'
|
|
1
|
+
import { execSync } from 'node:child_process';
|
|
2
|
+
export class RunProjectCommand {
|
|
3
|
+
command = 'run';
|
|
4
|
+
describe = 'Run crawlee project';
|
|
5
|
+
builder = async (args) => {
|
|
6
|
+
args.option('purge', {
|
|
7
|
+
alias: 't',
|
|
8
|
+
default: true,
|
|
9
|
+
type: 'boolean',
|
|
10
|
+
describe: 'Use `--no-purge` to disable automatic purging of default storages.',
|
|
12
11
|
});
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
value: 'Run crawlee project'
|
|
12
|
+
args.option('script', {
|
|
13
|
+
alias: 's',
|
|
14
|
+
default: 'start',
|
|
15
|
+
describe: 'Allows using different NPM script than `start`, e.g. `crawlee run --script=start:prod`.',
|
|
18
16
|
});
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
configurable: true,
|
|
22
|
-
writable: true,
|
|
23
|
-
value: async (args) => {
|
|
24
|
-
args.option('purge', {
|
|
25
|
-
alias: 't',
|
|
26
|
-
default: true,
|
|
27
|
-
type: 'boolean',
|
|
28
|
-
describe: 'Use `--no-purge` to disable automatic purging of default storages.',
|
|
29
|
-
});
|
|
30
|
-
args.option('script', {
|
|
31
|
-
alias: 's',
|
|
32
|
-
default: 'start',
|
|
33
|
-
describe: 'Allows using different NPM script than `start`, e.g. `crawlee run --script=start:prod`.',
|
|
34
|
-
});
|
|
35
|
-
return args;
|
|
36
|
-
}
|
|
37
|
-
});
|
|
38
|
-
}
|
|
17
|
+
return args;
|
|
18
|
+
};
|
|
39
19
|
/**
|
|
40
20
|
* @inheritDoc
|
|
41
21
|
*/
|
|
@@ -45,8 +25,7 @@ class RunProjectCommand {
|
|
|
45
25
|
env.CRAWLEE_PURGE_ON_START = '0';
|
|
46
26
|
}
|
|
47
27
|
// TODO detect the right package manager (e.g. based on package.json's `packageManager` field)
|
|
48
|
-
|
|
28
|
+
execSync(`npm run ${args.script}`, { stdio: 'inherit', env });
|
|
49
29
|
}
|
|
50
30
|
}
|
|
51
|
-
exports.RunProjectCommand = RunProjectCommand;
|
|
52
31
|
//# sourceMappingURL=RunProjectCommand.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"RunProjectCommand.js","sourceRoot":"","sources":["../../src/commands/RunProjectCommand.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"RunProjectCommand.js","sourceRoot":"","sources":["../../src/commands/RunProjectCommand.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAS9C,MAAM,OAAO,iBAAiB;IAC1B,OAAO,GAAG,KAAK,CAAC;IAChB,QAAQ,GAAG,qBAAqB,CAAC;IACjC,OAAO,GAAG,KAAK,EAAE,IAAa,EAAE,EAAE;QAC9B,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE;YACjB,KAAK,EAAE,GAAG;YACV,OAAO,EAAE,IAAI;YACb,IAAI,EAAE,SAAS;YACf,QAAQ,EAAE,oEAAoE;SACjF,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE;YAClB,KAAK,EAAE,GAAG;YACV,OAAO,EAAE,OAAO;YAChB,QAAQ,EAAE,yFAAyF;SACtG,CAAC,CAAC;QACH,OAAO,IAA4B,CAAC;IACxC,CAAC,CAAC;IAEF;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,IAAwC;QAClD,MAAM,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;QAE/B,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACd,GAAG,CAAC,sBAAsB,GAAG,GAAG,CAAC;QACrC,CAAC;QAED,8FAA8F;QAC9F,QAAQ,CAAC,WAAW,IAAI,CAAC,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;IAClE,CAAC;CACJ"}
|
package/index.js
CHANGED
|
@@ -1,21 +1,9 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
"use strict";
|
|
3
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
-
const tslib_1 = require("tslib");
|
|
5
|
-
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
6
|
-
require('yargonaut') //
|
|
7
|
-
.style('blue')
|
|
8
|
-
.style('yellow', 'required')
|
|
9
|
-
.helpStyle('green')
|
|
10
|
-
.errorsStyle('red');
|
|
11
2
|
// eslint-disable-next-line
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
const RunProjectCommand_1 = require("./commands/RunProjectCommand");
|
|
17
|
-
// eslint-disable-next-line
|
|
18
|
-
const yargs_1 = tslib_1.__importDefault(require("yargs"));
|
|
3
|
+
import { CreateProjectCommand } from './commands/CreateProjectCommand.js';
|
|
4
|
+
import { InstallPlaywrightBrowsersCommand } from './commands/InstallPlaywrightBrowsersCommand.js';
|
|
5
|
+
import { RunProjectCommand } from './commands/RunProjectCommand.js';
|
|
6
|
+
import yargs from 'yargs';
|
|
19
7
|
function getCLIVersion() {
|
|
20
8
|
try {
|
|
21
9
|
// this works during development (where we have `src` folder)
|
|
@@ -28,22 +16,24 @@ function getCLIVersion() {
|
|
|
28
16
|
return require('./package.json').version;
|
|
29
17
|
}
|
|
30
18
|
}
|
|
31
|
-
const cli =
|
|
19
|
+
const cli = yargs()
|
|
32
20
|
.scriptName('crawlee')
|
|
33
21
|
.version(getCLIVersion())
|
|
34
22
|
.usage('Usage: $0 <command> [options]')
|
|
35
23
|
.example('$0 run --no-purge', 'Runs the project in current working directory and disables automatic purging of default storages')
|
|
36
24
|
.alias('v', 'version')
|
|
37
25
|
.alias('h', 'help')
|
|
38
|
-
.command(new
|
|
39
|
-
.command(new
|
|
40
|
-
.command(new
|
|
26
|
+
.command(new CreateProjectCommand())
|
|
27
|
+
.command(new RunProjectCommand())
|
|
28
|
+
.command(new InstallPlaywrightBrowsersCommand())
|
|
41
29
|
.recommendCommands()
|
|
30
|
+
.showHelpOnFail(true)
|
|
31
|
+
.demandCommand(1, '')
|
|
42
32
|
.strict();
|
|
43
33
|
void (async () => {
|
|
44
34
|
const args = (await cli.parse(process.argv.slice(2)));
|
|
45
35
|
if (args._.length === 0) {
|
|
46
|
-
|
|
36
|
+
yargs(process.argv.slice(2)).showHelp();
|
|
47
37
|
}
|
|
48
38
|
})();
|
|
49
39
|
//# sourceMappingURL=index.js.map
|
package/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,2BAA2B;AAC3B,OAAO,EAAE,oBAAoB,EAAE,MAAM,oCAAoC,CAAC;AAE1E,OAAO,EAAE,gCAAgC,EAAE,MAAM,gDAAgD,CAAC;AAElG,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AAEpE,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,SAAS,aAAa;IAClB,IAAI,CAAC;QACD,6DAA6D;QAC7D,2BAA2B;QAC3B,OAAO,OAAO,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACL,yEAAyE;QACzE,2BAA2B;QAC3B,OAAO,OAAO,CAAC,gBAAgB,CAAC,CAAC,OAAO,CAAC;IAC7C,CAAC;AACL,CAAC;AAED,MAAM,GAAG,GAAG,KAAK,EAAE;KACd,UAAU,CAAC,SAAS,CAAC;KACrB,OAAO,CAAC,aAAa,EAAE,CAAC;KACxB,KAAK,CAAC,+BAA+B,CAAC;KACtC,OAAO,CACJ,mBAAmB,EACnB,kGAAkG,CACrG;KACA,KAAK,CAAC,GAAG,EAAE,SAAS,CAAC;KACrB,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC;KAClB,OAAO,CAAC,IAAI,oBAAoB,EAAE,CAAC;KACnC,OAAO,CAAC,IAAI,iBAAiB,EAAE,CAAC;KAChC,OAAO,CAAC,IAAI,gCAAgC,EAAE,CAAC;KAC/C,iBAAiB,EAAE;KACnB,cAAc,CAAC,IAAI,CAAC;KACpB,aAAa,CAAC,CAAC,EAAE,EAAE,CAAC;KACpB,MAAM,EAAE,CAAC;AAEd,KAAK,CAAC,KAAK,IAAI,EAAE;IACb,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAoB,CAAC;IAEzE,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;IAC5C,CAAC;AACL,CAAC,CAAC,EAAE,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,22 +1,16 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/cli",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "4.0.0-beta.0",
|
|
4
4
|
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
|
|
5
5
|
"engines": {
|
|
6
|
-
"node": ">=
|
|
6
|
+
"node": ">=22.0.0"
|
|
7
7
|
},
|
|
8
8
|
"bin": {
|
|
9
9
|
"crawlee": "./index.js"
|
|
10
10
|
},
|
|
11
|
-
"
|
|
12
|
-
"module": "./index.mjs",
|
|
13
|
-
"types": "./index.d.ts",
|
|
11
|
+
"type": "module",
|
|
14
12
|
"exports": {
|
|
15
|
-
".":
|
|
16
|
-
"import": "./index.mjs",
|
|
17
|
-
"require": "./index.js",
|
|
18
|
-
"types": "./index.d.ts"
|
|
19
|
-
},
|
|
13
|
+
".": "./index.js",
|
|
20
14
|
"./package.json": "./package.json"
|
|
21
15
|
},
|
|
22
16
|
"keywords": [
|
|
@@ -51,13 +45,12 @@
|
|
|
51
45
|
"access": "public"
|
|
52
46
|
},
|
|
53
47
|
"dependencies": {
|
|
54
|
-
"@crawlee/templates": "
|
|
48
|
+
"@crawlee/templates": "4.0.0-beta.0",
|
|
49
|
+
"@inquirer/prompts": "^7.5.0",
|
|
55
50
|
"ansi-colors": "^4.1.3",
|
|
56
|
-
"fs-extra": "^11.
|
|
57
|
-
"
|
|
58
|
-
"
|
|
59
|
-
"yargonaut": "^1.1.4",
|
|
60
|
-
"yargs": "^17.5.1"
|
|
51
|
+
"fs-extra": "^11.3.0",
|
|
52
|
+
"tslib": "^2.8.1",
|
|
53
|
+
"yargs": "^17.7.2"
|
|
61
54
|
},
|
|
62
55
|
"lerna": {
|
|
63
56
|
"command": {
|
|
@@ -66,5 +59,5 @@
|
|
|
66
59
|
}
|
|
67
60
|
}
|
|
68
61
|
},
|
|
69
|
-
"gitHead": "
|
|
62
|
+
"gitHead": "927bdafa403ff347327158b01d20b817378168a7"
|
|
70
63
|
}
|