apify-cli 1.0.0 → 1.1.1-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +242 -84
- package/LICENSE.md +201 -0
- package/README.md +49 -543
- package/dist/.tsbuildinfo +1 -1
- package/dist/actor.js +3 -0
- package/dist/actor.js.map +1 -0
- package/dist/apify.js +3 -0
- package/dist/apify.js.map +1 -0
- package/dist/chunk-C7RW4NOZ.js +242 -0
- package/dist/chunk-C7RW4NOZ.js.map +1 -0
- package/dist/index.js +2 -3
- package/package.json +113 -89
- package/bin/dev.cmd +0 -3
- package/bin/dev.js +0 -21
- package/bin/dev.sh +0 -4
- package/bin/run.cmd +0 -3
- package/bin/run.js +0 -23
- package/dist/commands/actor/get-input.d.ts +0 -6
- package/dist/commands/actor/get-input.d.ts.map +0 -1
- package/dist/commands/actor/get-input.js +0 -14
- package/dist/commands/actor/get-input.js.map +0 -1
- package/dist/commands/actor/get-value.d.ts +0 -9
- package/dist/commands/actor/get-value.d.ts.map +0 -1
- package/dist/commands/actor/get-value.js +0 -27
- package/dist/commands/actor/get-value.js.map +0 -1
- package/dist/commands/actor/index.d.ts +0 -6
- package/dist/commands/actor/index.d.ts.map +0 -1
- package/dist/commands/actor/index.js +0 -13
- package/dist/commands/actor/index.js.map +0 -1
- package/dist/commands/actor/push-data.d.ts +0 -11
- package/dist/commands/actor/push-data.d.ts.map +0 -1
- package/dist/commands/actor/push-data.js +0 -56
- package/dist/commands/actor/push-data.js.map +0 -1
- package/dist/commands/actor/set-value.d.ts +0 -15
- package/dist/commands/actor/set-value.d.ts.map +0 -1
- package/dist/commands/actor/set-value.js +0 -76
- package/dist/commands/actor/set-value.js.map +0 -1
- package/dist/commands/call.d.ts +0 -19
- package/dist/commands/call.d.ts.map +0 -1
- package/dist/commands/call.js +0 -193
- package/dist/commands/call.js.map +0 -1
- package/dist/commands/check-version.d.ts +0 -11
- package/dist/commands/check-version.d.ts.map +0 -1
- package/dist/commands/check-version.js +0 -39
- package/dist/commands/check-version.js.map +0 -1
- package/dist/commands/create.d.ts +0 -15
- package/dist/commands/create.d.ts.map +0 -1
- package/dist/commands/create.js +0 -201
- package/dist/commands/create.js.map +0 -1
- package/dist/commands/edit-input-schema.d.ts +0 -11
- package/dist/commands/edit-input-schema.d.ts.map +0 -1
- package/dist/commands/edit-input-schema.js +0 -176
- package/dist/commands/edit-input-schema.js.map +0 -1
- package/dist/commands/info.d.ts +0 -6
- package/dist/commands/info.d.ts.map +0 -1
- package/dist/commands/info.js +0 -26
- package/dist/commands/info.js.map +0 -1
- package/dist/commands/init-wrap-scrapy.d.ts +0 -10
- package/dist/commands/init-wrap-scrapy.d.ts.map +0 -1
- package/dist/commands/init-wrap-scrapy.js +0 -41
- package/dist/commands/init-wrap-scrapy.js.map +0 -1
- package/dist/commands/init.d.ts +0 -12
- package/dist/commands/init.d.ts.map +0 -1
- package/dist/commands/init.js +0 -88
- package/dist/commands/init.js.map +0 -1
- package/dist/commands/login.d.ts +0 -10
- package/dist/commands/login.d.ts.map +0 -1
- package/dist/commands/login.js +0 -174
- package/dist/commands/login.js.map +0 -1
- package/dist/commands/logout.d.ts +0 -6
- package/dist/commands/logout.d.ts.map +0 -1
- package/dist/commands/logout.js +0 -20
- package/dist/commands/logout.js.map +0 -1
- package/dist/commands/pull.d.ts +0 -12
- package/dist/commands/pull.d.ts.map +0 -1
- package/dist/commands/pull.js +0 -148
- package/dist/commands/pull.js.map +0 -1
- package/dist/commands/push.d.ts +0 -18
- package/dist/commands/push.d.ts.map +0 -1
- package/dist/commands/push.js +0 -298
- package/dist/commands/push.js.map +0 -1
- package/dist/commands/run.d.ts +0 -16
- package/dist/commands/run.d.ts.map +0 -1
- package/dist/commands/run.js +0 -383
- package/dist/commands/run.js.map +0 -1
- package/dist/commands/secrets/add.d.ts +0 -10
- package/dist/commands/secrets/add.d.ts.map +0 -1
- package/dist/commands/secrets/add.js +0 -31
- package/dist/commands/secrets/add.js.map +0 -1
- package/dist/commands/secrets/index.d.ts +0 -6
- package/dist/commands/secrets/index.d.ts.map +0 -1
- package/dist/commands/secrets/index.js +0 -24
- package/dist/commands/secrets/index.js.map +0 -1
- package/dist/commands/secrets/rm.d.ts +0 -9
- package/dist/commands/secrets/rm.d.ts.map +0 -1
- package/dist/commands/secrets/rm.js +0 -27
- package/dist/commands/secrets/rm.js.map +0 -1
- package/dist/commands/task/index.d.ts +0 -6
- package/dist/commands/task/index.d.ts.map +0 -1
- package/dist/commands/task/index.js +0 -13
- package/dist/commands/task/index.js.map +0 -1
- package/dist/commands/task/run.d.ts +0 -16
- package/dist/commands/task/run.d.ts.map +0 -1
- package/dist/commands/task/run.js +0 -91
- package/dist/commands/task/run.js.map +0 -1
- package/dist/commands/validate-schema.d.ts +0 -10
- package/dist/commands/validate-schema.d.ts.map +0 -1
- package/dist/commands/validate-schema.js +0 -58
- package/dist/commands/validate-schema.js.map +0 -1
- package/dist/hooks/deprecations.d.ts +0 -4
- package/dist/hooks/deprecations.d.ts.map +0 -1
- package/dist/hooks/deprecations.js +0 -18
- package/dist/hooks/deprecations.js.map +0 -1
- package/dist/hooks/init.d.ts +0 -7
- package/dist/hooks/init.d.ts.map +0 -1
- package/dist/hooks/init.js +0 -17
- package/dist/hooks/init.js.map +0 -1
- package/dist/index.d.ts +0 -2
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/lib/actor.d.ts +0 -27
- package/dist/lib/actor.d.ts.map +0 -1
- package/dist/lib/actor.js +0 -67
- package/dist/lib/actor.js.map +0 -1
- package/dist/lib/apify-oclif-help.d.ts +0 -15
- package/dist/lib/apify-oclif-help.d.ts.map +0 -1
- package/dist/lib/apify-oclif-help.js +0 -19
- package/dist/lib/apify-oclif-help.js.map +0 -1
- package/dist/lib/apify_command.d.ts +0 -21
- package/dist/lib/apify_command.d.ts.map +0 -1
- package/dist/lib/apify_command.js +0 -104
- package/dist/lib/apify_command.js.map +0 -1
- package/dist/lib/commands/resolve-input.d.ts +0 -5
- package/dist/lib/commands/resolve-input.d.ts.map +0 -1
- package/dist/lib/commands/resolve-input.js +0 -29
- package/dist/lib/commands/resolve-input.js.map +0 -1
- package/dist/lib/commands/run-on-cloud.d.ts +0 -20
- package/dist/lib/commands/run-on-cloud.d.ts.map +0 -1
- package/dist/lib/commands/run-on-cloud.js +0 -89
- package/dist/lib/commands/run-on-cloud.js.map +0 -1
- package/dist/lib/community.d.ts +0 -2
- package/dist/lib/community.d.ts.map +0 -1
- package/dist/lib/community.js +0 -4
- package/dist/lib/community.js.map +0 -1
- package/dist/lib/consts.d.ts +0 -54
- package/dist/lib/consts.d.ts.map +0 -1
- package/dist/lib/consts.js +0 -65
- package/dist/lib/consts.js.map +0 -1
- package/dist/lib/create-utils.d.ts +0 -8
- package/dist/lib/create-utils.d.ts.map +0 -1
- package/dist/lib/create-utils.js +0 -148
- package/dist/lib/create-utils.js.map +0 -1
- package/dist/lib/exec.d.ts +0 -3
- package/dist/lib/exec.d.ts.map +0 -1
- package/dist/lib/exec.js +0 -40
- package/dist/lib/exec.js.map +0 -1
- package/dist/lib/files.d.ts +0 -14
- package/dist/lib/files.d.ts.map +0 -1
- package/dist/lib/files.js +0 -58
- package/dist/lib/files.js.map +0 -1
- package/dist/lib/input_schema.d.ts +0 -28
- package/dist/lib/input_schema.d.ts.map +0 -1
- package/dist/lib/input_schema.js +0 -126
- package/dist/lib/input_schema.js.map +0 -1
- package/dist/lib/local_state.d.ts +0 -11
- package/dist/lib/local_state.d.ts.map +0 -1
- package/dist/lib/local_state.js +0 -27
- package/dist/lib/local_state.js.map +0 -1
- package/dist/lib/outputs.d.ts +0 -19
- package/dist/lib/outputs.d.ts.map +0 -1
- package/dist/lib/outputs.js +0 -45
- package/dist/lib/outputs.js.map +0 -1
- package/dist/lib/project_analyzer.d.ts +0 -4
- package/dist/lib/project_analyzer.d.ts.map +0 -1
- package/dist/lib/project_analyzer.js +0 -30
- package/dist/lib/project_analyzer.js.map +0 -1
- package/dist/lib/projects/CrawleeAnalyzer.d.ts +0 -4
- package/dist/lib/projects/CrawleeAnalyzer.d.ts.map +0 -1
- package/dist/lib/projects/CrawleeAnalyzer.js +0 -20
- package/dist/lib/projects/CrawleeAnalyzer.js.map +0 -1
- package/dist/lib/projects/OldApifySDKAnalyzer.d.ts +0 -4
- package/dist/lib/projects/OldApifySDKAnalyzer.d.ts.map +0 -1
- package/dist/lib/projects/OldApifySDKAnalyzer.js +0 -38
- package/dist/lib/projects/OldApifySDKAnalyzer.js.map +0 -1
- package/dist/lib/projects/scrapy/ScrapyProjectAnalyzer.d.ts +0 -17
- package/dist/lib/projects/scrapy/ScrapyProjectAnalyzer.d.ts.map +0 -1
- package/dist/lib/projects/scrapy/ScrapyProjectAnalyzer.js +0 -87
- package/dist/lib/projects/scrapy/ScrapyProjectAnalyzer.js.map +0 -1
- package/dist/lib/projects/scrapy/Spider.d.ts +0 -14
- package/dist/lib/projects/scrapy/Spider.d.ts.map +0 -1
- package/dist/lib/projects/scrapy/Spider.js +0 -33
- package/dist/lib/projects/scrapy/Spider.js.map +0 -1
- package/dist/lib/projects/scrapy/SpiderFileAnalyzer.d.ts +0 -7
- package/dist/lib/projects/scrapy/SpiderFileAnalyzer.d.ts.map +0 -1
- package/dist/lib/projects/scrapy/SpiderFileAnalyzer.js +0 -25
- package/dist/lib/projects/scrapy/SpiderFileAnalyzer.js.map +0 -1
- package/dist/lib/projects/scrapy/wrapScrapyProject.d.ts +0 -4
- package/dist/lib/projects/scrapy/wrapScrapyProject.d.ts.map +0 -1
- package/dist/lib/projects/scrapy/wrapScrapyProject.js +0 -109
- package/dist/lib/projects/scrapy/wrapScrapyProject.js.map +0 -1
- package/dist/lib/projects/shared.d.ts +0 -2
- package/dist/lib/projects/shared.d.ts.map +0 -1
- package/dist/lib/projects/shared.js +0 -13
- package/dist/lib/projects/shared.js.map +0 -1
- package/dist/lib/secrets.d.ts +0 -22
- package/dist/lib/secrets.d.ts.map +0 -1
- package/dist/lib/secrets.js +0 -104
- package/dist/lib/secrets.js.map +0 -1
- package/dist/lib/telemetry.d.ts +0 -21
- package/dist/lib/telemetry.d.ts.map +0 -1
- package/dist/lib/telemetry.js +0 -87
- package/dist/lib/telemetry.js.map +0 -1
- package/dist/lib/types.d.ts +0 -11
- package/dist/lib/types.d.ts.map +0 -1
- package/dist/lib/types.js +0 -2
- package/dist/lib/types.js.map +0 -1
- package/dist/lib/utils.d.ts +0 -102
- package/dist/lib/utils.d.ts.map +0 -1
- package/dist/lib/utils.js +0 -577
- package/dist/lib/utils.js.map +0 -1
- package/dist/lib/version_check.d.ts +0 -14
- package/dist/lib/version_check.d.ts.map +0 -1
- package/dist/lib/version_check.js +0 -90
- package/dist/lib/version_check.js.map +0 -1
- package/oclif.manifest.json +0 -867
package/dist/lib/input_schema.js
DELETED
|
@@ -1,126 +0,0 @@
|
|
|
1
|
-
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
2
|
-
import { existsSync } from 'node:fs';
|
|
3
|
-
import { join } from 'node:path';
|
|
4
|
-
import process from 'node:process';
|
|
5
|
-
import { KEY_VALUE_STORE_KEYS } from '@apify/consts';
|
|
6
|
-
import { validateInputSchema } from '@apify/input_schema';
|
|
7
|
-
import deepClone from 'lodash.clonedeep';
|
|
8
|
-
import _ from 'underscore';
|
|
9
|
-
import { writeJsonFile } from 'write-json-file';
|
|
10
|
-
import { ACTOR_SPECIFICATION_FOLDER } from './consts.js';
|
|
11
|
-
import { warning } from './outputs.js';
|
|
12
|
-
import { Ajv, getJsonFileContent, getLocalConfig, getLocalKeyValueStorePath } from './utils.js';
|
|
13
|
-
const DEFAULT_INPUT_SCHEMA_PATHS = [
|
|
14
|
-
'.actor/INPUT_SCHEMA.json',
|
|
15
|
-
'./INPUT_SCHEMA.json',
|
|
16
|
-
];
|
|
17
|
-
/**
|
|
18
|
-
* Return the input schema from the default location.
|
|
19
|
-
*
|
|
20
|
-
* When the input schema does not exist, null is returned for schema.
|
|
21
|
-
* In such a case, path would be set to the location
|
|
22
|
-
* where the input schema would be expected to be found (and e.g. can be created there).
|
|
23
|
-
*/
|
|
24
|
-
export const readInputSchema = async ({ forcePath, cwd } = { cwd: process.cwd() }) => {
|
|
25
|
-
if (forcePath) {
|
|
26
|
-
return {
|
|
27
|
-
inputSchema: getJsonFileContent(forcePath),
|
|
28
|
-
inputSchemaPath: forcePath,
|
|
29
|
-
};
|
|
30
|
-
}
|
|
31
|
-
const localConfig = getLocalConfig(cwd);
|
|
32
|
-
if (typeof localConfig?.input === 'object') {
|
|
33
|
-
return {
|
|
34
|
-
inputSchema: localConfig.input,
|
|
35
|
-
inputSchemaPath: null,
|
|
36
|
-
};
|
|
37
|
-
}
|
|
38
|
-
if (typeof localConfig?.input === 'string') {
|
|
39
|
-
const fullPath = join(cwd, ACTOR_SPECIFICATION_FOLDER, localConfig.input);
|
|
40
|
-
return {
|
|
41
|
-
inputSchema: getJsonFileContent(fullPath),
|
|
42
|
-
inputSchemaPath: fullPath,
|
|
43
|
-
};
|
|
44
|
-
}
|
|
45
|
-
if (existsSync(join(cwd, DEFAULT_INPUT_SCHEMA_PATHS[0]))) {
|
|
46
|
-
return {
|
|
47
|
-
inputSchema: getJsonFileContent(join(cwd, DEFAULT_INPUT_SCHEMA_PATHS[0])),
|
|
48
|
-
inputSchemaPath: join(cwd, DEFAULT_INPUT_SCHEMA_PATHS[0]),
|
|
49
|
-
};
|
|
50
|
-
}
|
|
51
|
-
if (existsSync(join(cwd, DEFAULT_INPUT_SCHEMA_PATHS[1]))) {
|
|
52
|
-
return {
|
|
53
|
-
inputSchema: getJsonFileContent(join(cwd, DEFAULT_INPUT_SCHEMA_PATHS[1])),
|
|
54
|
-
inputSchemaPath: join(cwd, DEFAULT_INPUT_SCHEMA_PATHS[1]),
|
|
55
|
-
};
|
|
56
|
-
}
|
|
57
|
-
// If not input schema has been found so far, return the first default path
|
|
58
|
-
// where the input schema would be expected.
|
|
59
|
-
return {
|
|
60
|
-
inputSchema: null,
|
|
61
|
-
inputSchemaPath: join(cwd, DEFAULT_INPUT_SCHEMA_PATHS[0]),
|
|
62
|
-
};
|
|
63
|
-
};
|
|
64
|
-
/**
|
|
65
|
-
* Goes to the Actor directory and creates INPUT.json file from the input schema prefills.
|
|
66
|
-
|
|
67
|
-
*/
|
|
68
|
-
export const createPrefilledInputFileFromInputSchema = async (actorFolderDir) => {
|
|
69
|
-
let inputFile = {};
|
|
70
|
-
try {
|
|
71
|
-
const { inputSchema } = await readInputSchema({ cwd: actorFolderDir });
|
|
72
|
-
if (inputSchema) {
|
|
73
|
-
/**
|
|
74
|
-
* TODO: The logic is copied from @apify-packages/actor -> getPrefillFromInputSchema
|
|
75
|
-
* It is not possible to install the package here because it is private
|
|
76
|
-
* We should move it to @apify/input_schema and use it from there.
|
|
77
|
-
*/
|
|
78
|
-
const validator = new Ajv({ strict: false });
|
|
79
|
-
validateInputSchema(validator, inputSchema);
|
|
80
|
-
inputFile = _.mapObject(inputSchema.properties, (fieldSchema) => ((fieldSchema.type === 'boolean' || fieldSchema.editor === 'hidden')
|
|
81
|
-
? fieldSchema.default
|
|
82
|
-
: fieldSchema.prefill));
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
catch (err) {
|
|
86
|
-
warning({ message: `Could not create default input based on input schema, creating empty input instead. Cause: ${err.message}` });
|
|
87
|
-
}
|
|
88
|
-
finally {
|
|
89
|
-
const keyValueStorePath = getLocalKeyValueStorePath();
|
|
90
|
-
const inputJsonPath = join(actorFolderDir, keyValueStorePath, `${KEY_VALUE_STORE_KEYS.INPUT}.json`);
|
|
91
|
-
await writeJsonFile(inputJsonPath, inputFile);
|
|
92
|
-
}
|
|
93
|
-
};
|
|
94
|
-
export const getDefaultsAndPrefillsFromInputSchema = (inputSchema) => {
|
|
95
|
-
const defaults = {};
|
|
96
|
-
for (const [key, fieldSchema] of Object.entries(inputSchema.properties)) {
|
|
97
|
-
if (fieldSchema.default !== undefined) {
|
|
98
|
-
defaults[key] = fieldSchema.default;
|
|
99
|
-
}
|
|
100
|
-
else if (fieldSchema.prefill !== undefined) {
|
|
101
|
-
defaults[key] = fieldSchema.prefill;
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
|
-
return defaults;
|
|
105
|
-
};
|
|
106
|
-
// Lots of code copied from @apify-packages/actor, this really should be moved to the shared input_schema package
|
|
107
|
-
export const getAjvValidator = (inputSchema, ajvInstance) => {
|
|
108
|
-
const copyOfSchema = deepClone(inputSchema);
|
|
109
|
-
copyOfSchema.required = [];
|
|
110
|
-
for (const [inputSchemaFieldKey, inputSchemaField] of Object.entries(inputSchema.properties)) {
|
|
111
|
-
// `required` field doesn't need to be present in input schema
|
|
112
|
-
const isRequired = inputSchema.required?.includes(inputSchemaFieldKey);
|
|
113
|
-
const hasDefault = inputSchemaField.default !== undefined;
|
|
114
|
-
if (isRequired && !hasDefault) {
|
|
115
|
-
// If field is required but has default, we act like it's optional because we always have value to use
|
|
116
|
-
copyOfSchema.required.push(inputSchemaFieldKey);
|
|
117
|
-
if (inputSchemaField.type === 'array') {
|
|
118
|
-
// If array is required, it has to have at least 1 item.
|
|
119
|
-
inputSchemaField.minItems = Math.max(1, inputSchemaField.minItems || 0);
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
delete copyOfSchema.$schema;
|
|
124
|
-
return ajvInstance.compile(copyOfSchema);
|
|
125
|
-
};
|
|
126
|
-
//# sourceMappingURL=input_schema.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"input_schema.js","sourceRoot":"","sources":["../../src/lib/input_schema.ts"],"names":[],"mappings":"AAAA,uDAAuD;AACvD,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,OAAO,MAAM,cAAc,CAAC;AAEnC,OAAO,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AACrD,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,SAAS,MAAM,kBAAkB,CAAC;AACzC,OAAO,CAAC,MAAM,YAAY,CAAC;AAC3B,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEhD,OAAO,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,GAAG,EAAE,kBAAkB,EAAE,cAAc,EAAE,yBAAyB,EAAE,MAAM,YAAY,CAAC;AAEhG,MAAM,0BAA0B,GAAG;IAC/B,0BAA0B;IAC1B,qBAAqB;CACxB,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,EAAE,EAAE,SAAS,EAAE,GAAG,KAA0C,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE;IACtH,IAAI,SAAS,EAAE,CAAC;QACZ,OAAO;YACH,WAAW,EAAE,kBAAkB,CAAC,SAAS,CAAC;YAC1C,eAAe,EAAE,SAAS;SAC7B,CAAC;IACN,CAAC;IAED,MAAM,WAAW,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;IAExC,IAAI,OAAO,WAAW,EAAE,KAAK,KAAK,QAAQ,EAAE,CAAC;QACzC,OAAO;YACH,WAAW,EAAE,WAAW,CAAC,KAAgC;YACzD,eAAe,EAAE,IAAI;SACxB,CAAC;IACN,CAAC;IAED,IAAI,OAAO,WAAW,EAAE,KAAK,KAAK,QAAQ,EAAE,CAAC;QACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,0BAA0B,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC;QAC1E,OAAO;YACH,WAAW,EAAE,kBAAkB,CAAC,QAAQ,CAAC;YACzC,eAAe,EAAE,QAAQ;SAC5B,CAAC;IACN,CAAC;IAED,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,0BAA0B,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACvD,OAAO;YACH,WAAW,EAAE,kBAAkB,CAAC,IAAI,CAAC,GAAG,EAAE,0BAA0B,CAAC,CAAC,CAAC,CAAC,CAAC;YACzE,eAAe,EAAE,IAAI,CAAC,GAAG,EAAE,0BAA0B,CAAC,CAAC,CAAC,CAAC;SAC5D,CAAC;IACN,CAAC;IAED,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,0BAA0B,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACvD,OAAO;YACH,WAAW,EAAE,kBAAkB,CAAC,IAAI,CAAC,GAAG,EAAE,0BAA0B,CAAC,CAAC,CAAC,CAAC,CAAC;YACzE,eAAe,EAAE,IAAI,CAAC,GAAG,EAAE,0BAA0B,CAAC,CAAC,CAAC,CAAC;SAC5D,CAAC;IACN,CAAC;IAED,2EAA2E;IAC3E,4CAA4C;IAC5C,OAAO;QACH,WAAW,EAAE,IAAI;QACjB,eAAe,EAAE,IAAI,CAAC,GAAG,EAAE,0BAA0B,CAAC,CAAC,CAAC,CAAC;KAC5D,CAAC;AACN,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,uCAAuC,GAAG,KAAK,EAAE,cAAsB,EAAE,EAAE;IACpF,IAAI,SAAS,GAAG,EAAE,CAAC;IACnB,IAAI,CAAC;QACD,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,eAAe,CAAC,EAAE,GAAG,EAAE,cAAc,EAAE,CAAC,CAAC;QAEvE,IAAI,WAAW,EAAE,CAAC;YACd;;;;eAIG;YACH,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;YAC7C,mBAAmB,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;YAE5C,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,WAAW,CAAC,UAAiB,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,KAAK,SAAS,IAAI,WAAW,CAAC,MAAM,KAAK,QAAQ,CAAC;gBACxI,CAAC,CAAC,WAAW,CAAC,OAAO;gBACrB,CAAC,CAAC,WAAW,CAAC,OAAO,CACxB,CAAC,CAAC;QACP,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,OAAO,CAAC,EAAE,OAAO,EAAE,8FAA+F,GAAa,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IACjJ,CAAC;YAAS,CAAC;QACP,MAAM,iBAAiB,GAAG,yBAAyB,EAAE,CAAC;QACtD,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,EAAE,iBAAiB,EAAE,GAAG,oBAAoB,CAAC,KAAK,OAAO,CAAC,CAAC;QACpG,MAAM,aAAa,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;IAClD,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,qCAAqC,GAAG,CAAC,WAAgB,EAAE,EAAE;IACtE,MAAM,QAAQ,GAA4B,EAAE,CAAC;IAE7C,KAAK,MAAM,CAAC,GAAG,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,OAAO,CAAM,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3E,IAAI,WAAW,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YACpC,QAAQ,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,OAAO,CAAC;QACxC,CAAC;aAAM,IAAI,WAAW,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC3C,QAAQ,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,OAAO,CAAC;QACxC,CAAC;IACL,CAAC;IAED,OAAO,QAAQ,CAAC;AACpB,CAAC,CAAC;AAEF,iHAAiH;AACjH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,WAAgB,EAAE,WAA8B,EAAE,EAAE;IAChF,MAAM,YAAY,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC;IAC5C,YAAY,CAAC,QAAQ,GAAG,EAAE,CAAC;IAE3B,KAAK,MAAM,CAAC,mBAAmB,EAAE,gBAAgB,CAAC,IAAI,MAAM,CAAC,OAAO,CAAM,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC;QAChG,8DAA8D;QAC9D,MAAM,UAAU,GAAG,WAAW,CAAC,QAAQ,EAAE,QAAQ,CAAC,mBAAmB,CAAC,CAAC;QACvE,MAAM,UAAU,GAAG,gBAAgB,CAAC,OAAO,KAAK,SAAS,CAAC;QAE1D,IAAI,UAAU,IAAI,CAAC,UAAU,EAAE,CAAC;YAC5B,sGAAsG;YACtG,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;YAChD,IAAI,gBAAgB,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;gBACpC,wDAAwD;gBACxD,gBAAgB,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,gBAAgB,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;YAC5E,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,YAAY,CAAC,OAAO,CAAC;IAE5B,OAAO,WAAW,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;AAC7C,CAAC,CAAC"}
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Returns state object from auth file or empty object.
|
|
3
|
-
* This method is synchronous/blocking to avoid different race conditions.
|
|
4
|
-
*/
|
|
5
|
-
export declare const getLocalState: () => Record<string, unknown>;
|
|
6
|
-
/**
|
|
7
|
-
* Extends local state by given values.
|
|
8
|
-
* This method is synchronous/blocking to avoid different race conditions.
|
|
9
|
-
*/
|
|
10
|
-
export declare const extendLocalState: (data: Record<string, unknown>) => void;
|
|
11
|
-
//# sourceMappingURL=local_state.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"local_state.d.ts","sourceRoot":"","sources":["../../src/lib/local_state.ts"],"names":[],"mappings":"AAOA;;;GAGG;AACH,eAAO,MAAM,aAAa,+BAMzB,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,gBAAgB,SAAU,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,SAM7D,CAAC"}
|
package/dist/lib/local_state.js
DELETED
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
import { loadJsonFileSync } from 'load-json-file';
|
|
2
|
-
import { writeJsonFileSync } from 'write-json-file';
|
|
3
|
-
import { STATE_FILE_PATH, } from './consts.js';
|
|
4
|
-
/**
|
|
5
|
-
* Returns state object from auth file or empty object.
|
|
6
|
-
* This method is synchronous/blocking to avoid different race conditions.
|
|
7
|
-
*/
|
|
8
|
-
export const getLocalState = () => {
|
|
9
|
-
try {
|
|
10
|
-
return loadJsonFileSync(STATE_FILE_PATH()) || {};
|
|
11
|
-
}
|
|
12
|
-
catch (e) {
|
|
13
|
-
return {};
|
|
14
|
-
}
|
|
15
|
-
};
|
|
16
|
-
/**
|
|
17
|
-
* Extends local state by given values.
|
|
18
|
-
* This method is synchronous/blocking to avoid different race conditions.
|
|
19
|
-
*/
|
|
20
|
-
export const extendLocalState = (data) => {
|
|
21
|
-
const state = getLocalState();
|
|
22
|
-
writeJsonFileSync(STATE_FILE_PATH(), {
|
|
23
|
-
...state,
|
|
24
|
-
...data,
|
|
25
|
-
});
|
|
26
|
-
};
|
|
27
|
-
//# sourceMappingURL=local_state.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"local_state.js","sourceRoot":"","sources":["../../src/lib/local_state.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEpD,OAAO,EACH,eAAe,GAClB,MAAM,aAAa,CAAC;AAErB;;;GAGG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,GAAG,EAAE;IAC9B,IAAI,CAAC;QACD,OAAO,gBAAgB,CAA0B,eAAe,EAAE,CAAC,IAAI,EAAE,CAAC;IAC9E,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,OAAO,EAAE,CAAC;IACd,CAAC;AACL,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,IAA6B,EAAE,EAAE;IAC9D,MAAM,KAAK,GAAG,aAAa,EAAE,CAAC;IAC9B,iBAAiB,CAAC,eAAe,EAAE,EAAE;QACjC,GAAG,KAAK;QACR,GAAG,IAAI;KACV,CAAC,CAAC;AACP,CAAC,CAAC"}
|
package/dist/lib/outputs.d.ts
DELETED
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
export interface LogOptions {
|
|
2
|
-
stdoutOutput?: unknown[];
|
|
3
|
-
stderrOutput?: unknown[];
|
|
4
|
-
}
|
|
5
|
-
export interface SimpleLogOptions {
|
|
6
|
-
stdout?: boolean;
|
|
7
|
-
message: string;
|
|
8
|
-
}
|
|
9
|
-
export declare function simpleLog(options: SimpleLogOptions): void;
|
|
10
|
-
export declare function error(options: SimpleLogOptions): void;
|
|
11
|
-
export declare function warning(options: SimpleLogOptions): void;
|
|
12
|
-
export declare function success(options: SimpleLogOptions): void;
|
|
13
|
-
export declare function run(options: SimpleLogOptions): void;
|
|
14
|
-
export declare function info(options: SimpleLogOptions): void;
|
|
15
|
-
export interface SimpleLinkOptions extends SimpleLogOptions {
|
|
16
|
-
url: string;
|
|
17
|
-
}
|
|
18
|
-
export declare function link(options: SimpleLinkOptions): void;
|
|
19
|
-
//# sourceMappingURL=outputs.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"outputs.d.ts","sourceRoot":"","sources":["../../src/lib/outputs.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,UAAU;IACvB,YAAY,CAAC,EAAE,OAAO,EAAE,CAAC;IACzB,YAAY,CAAC,EAAE,OAAO,EAAE,CAAC;CAC5B;AAYD,MAAM,WAAW,gBAAgB;IAC7B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACnB;AAED,wBAAgB,SAAS,CAAC,OAAO,EAAE,gBAAgB,QAIlD;AAED,wBAAgB,KAAK,CAAC,OAAO,EAAE,gBAAgB,QAI9C;AAED,wBAAgB,OAAO,CAAC,OAAO,EAAE,gBAAgB,QAIhD;AAED,wBAAgB,OAAO,CAAC,OAAO,EAAE,gBAAgB,QAIhD;AAED,wBAAgB,GAAG,CAAC,OAAO,EAAE,gBAAgB,QAI5C;AAED,wBAAgB,IAAI,CAAC,OAAO,EAAE,gBAAgB,QAI7C;AAED,MAAM,WAAW,iBAAkB,SAAQ,gBAAgB;IACvD,GAAG,EAAE,MAAM,CAAC;CACf;AAED,wBAAgB,IAAI,CAAC,OAAO,EAAE,iBAAiB,QAI9C"}
|
package/dist/lib/outputs.js
DELETED
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
import chalk from 'chalk';
|
|
2
|
-
function internalLog(options) {
|
|
3
|
-
if (options.stdoutOutput) {
|
|
4
|
-
console.log(...options.stdoutOutput);
|
|
5
|
-
}
|
|
6
|
-
if (options.stderrOutput) {
|
|
7
|
-
console.error(...options.stderrOutput);
|
|
8
|
-
}
|
|
9
|
-
}
|
|
10
|
-
export function simpleLog(options) {
|
|
11
|
-
internalLog({
|
|
12
|
-
[options.stdout ? 'stdoutOutput' : 'stderrOutput']: [options.message],
|
|
13
|
-
});
|
|
14
|
-
}
|
|
15
|
-
export function error(options) {
|
|
16
|
-
internalLog({
|
|
17
|
-
[options.stdout ? 'stdoutOutput' : 'stderrOutput']: [chalk.red('Error:'), options.message],
|
|
18
|
-
});
|
|
19
|
-
}
|
|
20
|
-
export function warning(options) {
|
|
21
|
-
internalLog({
|
|
22
|
-
[options.stdout ? 'stdoutOutput' : 'stderrOutput']: [chalk.rgb(254, 90, 29).bold('Warning:'), options.message],
|
|
23
|
-
});
|
|
24
|
-
}
|
|
25
|
-
export function success(options) {
|
|
26
|
-
internalLog({
|
|
27
|
-
[options.stdout ? 'stdoutOutput' : 'stderrOutput']: [chalk.green('Success:'), options.message],
|
|
28
|
-
});
|
|
29
|
-
}
|
|
30
|
-
export function run(options) {
|
|
31
|
-
internalLog({
|
|
32
|
-
[options.stdout ? 'stdoutOutput' : 'stderrOutput']: [chalk.gray('Run:'), options.message],
|
|
33
|
-
});
|
|
34
|
-
}
|
|
35
|
-
export function info(options) {
|
|
36
|
-
internalLog({
|
|
37
|
-
[options.stdout ? 'stdoutOutput' : 'stderrOutput']: [chalk.white('Info:'), options.message],
|
|
38
|
-
});
|
|
39
|
-
}
|
|
40
|
-
export function link(options) {
|
|
41
|
-
internalLog({
|
|
42
|
-
[options.stdout ? 'stdoutOutput' : 'stderrOutput']: [chalk.blue(options.message), options.url],
|
|
43
|
-
});
|
|
44
|
-
}
|
|
45
|
-
//# sourceMappingURL=outputs.js.map
|
package/dist/lib/outputs.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"outputs.js","sourceRoot":"","sources":["../../src/lib/outputs.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAO1B,SAAS,WAAW,CAAC,OAAmB;IACpC,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QACvB,OAAO,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QACvB,OAAO,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IAC3C,CAAC;AACL,CAAC;AAOD,MAAM,UAAU,SAAS,CAAC,OAAyB;IAC/C,WAAW,CAAC;QACR,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC;KACxE,CAAC,CAAC;AACP,CAAC;AAED,MAAM,UAAU,KAAK,CAAC,OAAyB;IAC3C,WAAW,CAAC;QACR,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC;KAC7F,CAAC,CAAC;AACP,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,OAAyB;IAC7C,WAAW,CAAC;QACR,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC;KACjH,CAAC,CAAC;AACP,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,OAAyB;IAC7C,WAAW,CAAC;QACR,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC;KACjG,CAAC,CAAC;AACP,CAAC;AAED,MAAM,UAAU,GAAG,CAAC,OAAyB;IACzC,WAAW,CAAC;QACR,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC;KAC5F,CAAC,CAAC;AACP,CAAC;AAED,MAAM,UAAU,IAAI,CAAC,OAAyB;IAC1C,WAAW,CAAC;QACR,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC;KAC9F,CAAC,CAAC;AACP,CAAC;AAMD,MAAM,UAAU,IAAI,CAAC,OAA0B;IAC3C,WAAW,CAAC;QACR,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,GAAG,CAAC;KACjG,CAAC,CAAC;AACP,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"project_analyzer.d.ts","sourceRoot":"","sources":["../../src/lib/project_analyzer.ts"],"names":[],"mappings":"AA4BA,qBAAa,eAAe;IACxB,MAAM,CAAC,cAAc,CAAC,QAAQ,EAAE,MAAM;CAWzC"}
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
import { PROJECT_TYPES } from './consts.js';
|
|
2
|
-
import { CrawleeAnalyzer } from './projects/CrawleeAnalyzer.js';
|
|
3
|
-
import { OldApifySDKAnalyzer } from './projects/OldApifySDKAnalyzer.js';
|
|
4
|
-
import { ScrapyProjectAnalyzer } from './projects/scrapy/ScrapyProjectAnalyzer.js';
|
|
5
|
-
const analyzers = [
|
|
6
|
-
{
|
|
7
|
-
type: PROJECT_TYPES.SCRAPY,
|
|
8
|
-
analyzer: ScrapyProjectAnalyzer,
|
|
9
|
-
},
|
|
10
|
-
{
|
|
11
|
-
type: PROJECT_TYPES.CRAWLEE,
|
|
12
|
-
analyzer: CrawleeAnalyzer,
|
|
13
|
-
},
|
|
14
|
-
{
|
|
15
|
-
type: PROJECT_TYPES.PRE_CRAWLEE_APIFY_SDK,
|
|
16
|
-
analyzer: OldApifySDKAnalyzer,
|
|
17
|
-
},
|
|
18
|
-
];
|
|
19
|
-
export class ProjectAnalyzer {
|
|
20
|
-
static getProjectType(pathname) {
|
|
21
|
-
const analyzer = analyzers.find((a) => {
|
|
22
|
-
if (!a.analyzer.isApplicable) {
|
|
23
|
-
throw new Error(`Analyzer ${a.analyzer} does not have isApplicable method.`);
|
|
24
|
-
}
|
|
25
|
-
return a.analyzer.isApplicable(pathname);
|
|
26
|
-
});
|
|
27
|
-
return analyzer?.type || PROJECT_TYPES.UNKNOWN;
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
//# sourceMappingURL=project_analyzer.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"project_analyzer.js","sourceRoot":"","sources":["../../src/lib/project_analyzer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AACxE,OAAO,EAAE,qBAAqB,EAAE,MAAM,4CAA4C,CAAC;AAUnF,MAAM,SAAS,GAA0B;IACrC;QACI,IAAI,EAAE,aAAa,CAAC,MAAM;QAC1B,QAAQ,EAAE,qBAAqB;KAClC;IACD;QACI,IAAI,EAAE,aAAa,CAAC,OAAO;QAC3B,QAAQ,EAAE,eAAe;KAC5B;IACD;QACI,IAAI,EAAE,aAAa,CAAC,qBAAqB;QACzC,QAAQ,EAAE,mBAAmB;KAChC;CACJ,CAAC;AAEF,MAAM,OAAO,eAAe;IACxB,MAAM,CAAC,cAAc,CAAC,QAAgB;QAClC,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;YAClC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY,EAAE,CAAC;gBAC3B,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,CAAC,QAAQ,qCAAqC,CAAC,CAAC;YACjF,CAAC;YAED,OAAO,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;QAC7C,CAAC,CAAC,CAAC;QAEH,OAAO,QAAQ,EAAE,IAAI,IAAI,aAAa,CAAC,OAAO,CAAC;IACnD,CAAC;CACJ"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"CrawleeAnalyzer.d.ts","sourceRoot":"","sources":["../../../src/lib/projects/CrawleeAnalyzer.ts"],"names":[],"mappings":"AAKA,qBAAa,eAAe;IACxB,MAAM,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM;CAiBvC"}
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
import { existsSync, readFileSync } from 'node:fs';
|
|
2
|
-
import { join } from 'node:path';
|
|
3
|
-
import { CRAWLEE_PACKAGES } from './shared.js';
|
|
4
|
-
export class CrawleeAnalyzer {
|
|
5
|
-
static isApplicable(pathname) {
|
|
6
|
-
const hasPackageJson = existsSync(join(pathname, 'package.json'));
|
|
7
|
-
if (!hasPackageJson) {
|
|
8
|
-
return false;
|
|
9
|
-
}
|
|
10
|
-
const packageJson = readFileSync(join(pathname, 'package.json'), 'utf8');
|
|
11
|
-
try {
|
|
12
|
-
const packageJsonParsed = JSON.parse(packageJson);
|
|
13
|
-
return CRAWLEE_PACKAGES.some((pkg) => packageJsonParsed?.dependencies?.[pkg] !== undefined);
|
|
14
|
-
}
|
|
15
|
-
catch (err) {
|
|
16
|
-
return false;
|
|
17
|
-
}
|
|
18
|
-
}
|
|
19
|
-
}
|
|
20
|
-
//# sourceMappingURL=CrawleeAnalyzer.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"CrawleeAnalyzer.js","sourceRoot":"","sources":["../../../src/lib/projects/CrawleeAnalyzer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAE/C,MAAM,OAAO,eAAe;IACxB,MAAM,CAAC,YAAY,CAAC,QAAgB;QAChC,MAAM,cAAc,GAAG,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC,CAAC;QAElE,IAAI,CAAC,cAAc,EAAE,CAAC;YAClB,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,CAAC;QAEzE,IAAI,CAAC;YACD,MAAM,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YAElD,OAAO,gBAAgB,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,iBAAiB,EAAE,YAAY,EAAE,CAAC,GAAG,CAAC,KAAK,SAAS,CAAC,CAAC;QAChG,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;CACJ"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"OldApifySDKAnalyzer.d.ts","sourceRoot":"","sources":["../../../src/lib/projects/OldApifySDKAnalyzer.ts"],"names":[],"mappings":"AASA,qBAAa,mBAAmB;IAC5B,MAAM,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM;CAsCvC"}
|
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
import { existsSync, readFileSync } from 'node:fs';
|
|
2
|
-
import { join } from 'node:path';
|
|
3
|
-
import { lt } from 'semver';
|
|
4
|
-
import { CRAWLEE_PACKAGES } from './shared.js';
|
|
5
|
-
const VERSION_WHEN_APIFY_MOVED_TO_CRAWLEE = '3.0.0';
|
|
6
|
-
export class OldApifySDKAnalyzer {
|
|
7
|
-
static isApplicable(pathname) {
|
|
8
|
-
const hasPackageJson = existsSync(join(pathname, 'package.json'));
|
|
9
|
-
if (!hasPackageJson) {
|
|
10
|
-
return false;
|
|
11
|
-
}
|
|
12
|
-
const packageJson = readFileSync(join(pathname, 'package.json'), 'utf8');
|
|
13
|
-
try {
|
|
14
|
-
const packageJsonParsed = JSON.parse(packageJson);
|
|
15
|
-
// If they have crawlee as a dependency, likely to use crawlee
|
|
16
|
-
if (CRAWLEE_PACKAGES.some((pkg) => packageJsonParsed?.dependencies?.[pkg] !== undefined)) {
|
|
17
|
-
return false;
|
|
18
|
-
}
|
|
19
|
-
const apifyVersion = packageJsonParsed?.dependencies?.apify;
|
|
20
|
-
if (!apifyVersion) {
|
|
21
|
-
return false;
|
|
22
|
-
}
|
|
23
|
-
// We cannot infer/crawlee v3
|
|
24
|
-
if (apifyVersion === '*') {
|
|
25
|
-
return false;
|
|
26
|
-
}
|
|
27
|
-
let actualVersion = apifyVersion;
|
|
28
|
-
if (apifyVersion.startsWith('~') || apifyVersion.startsWith('^')) {
|
|
29
|
-
actualVersion = apifyVersion.slice(1);
|
|
30
|
-
}
|
|
31
|
-
return lt(actualVersion, VERSION_WHEN_APIFY_MOVED_TO_CRAWLEE);
|
|
32
|
-
}
|
|
33
|
-
catch (err) {
|
|
34
|
-
return false;
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
//# sourceMappingURL=OldApifySDKAnalyzer.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"OldApifySDKAnalyzer.js","sourceRoot":"","sources":["../../../src/lib/projects/OldApifySDKAnalyzer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAE/C,MAAM,mCAAmC,GAAG,OAAO,CAAC;AAEpD,MAAM,OAAO,mBAAmB;IAC5B,MAAM,CAAC,YAAY,CAAC,QAAgB;QAChC,MAAM,cAAc,GAAG,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC,CAAC;QAElE,IAAI,CAAC,cAAc,EAAE,CAAC;YAClB,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,CAAC;QAEzE,IAAI,CAAC;YACD,MAAM,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YAElD,8DAA8D;YAC9D,IAAI,gBAAgB,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,iBAAiB,EAAE,YAAY,EAAE,CAAC,GAAG,CAAC,KAAK,SAAS,CAAC,EAAE,CAAC;gBACvF,OAAO,KAAK,CAAC;YACjB,CAAC;YAED,MAAM,YAAY,GAAG,iBAAiB,EAAE,YAAY,EAAE,KAAK,CAAC;YAC5D,IAAI,CAAC,YAAY,EAAE,CAAC;gBAChB,OAAO,KAAK,CAAC;YACjB,CAAC;YAED,6BAA6B;YAC7B,IAAI,YAAY,KAAK,GAAG,EAAE,CAAC;gBACvB,OAAO,KAAK,CAAC;YACjB,CAAC;YAED,IAAI,aAAa,GAAG,YAAY,CAAC;YAEjC,IAAI,YAAY,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,YAAY,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC/D,aAAa,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC1C,CAAC;YAED,OAAO,EAAE,CAAC,aAAa,EAAE,mCAAmC,CAAC,CAAC;QAClE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;CACJ"}
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
import ConfigParser from 'configparser';
|
|
2
|
-
export declare class ScrapyProjectAnalyzer {
|
|
3
|
-
pathname: string;
|
|
4
|
-
configuration: ConfigParser;
|
|
5
|
-
settings: {
|
|
6
|
-
BOT_NAME: string;
|
|
7
|
-
SPIDER_MODULES: string[];
|
|
8
|
-
} | null;
|
|
9
|
-
constructor(pathname: string);
|
|
10
|
-
static isApplicable(pathname: string): boolean;
|
|
11
|
-
init(): Promise<void>;
|
|
12
|
-
loadScrapyCfg(): void;
|
|
13
|
-
loadSettings(): Promise<void>;
|
|
14
|
-
getName(): string | undefined;
|
|
15
|
-
getAvailableSpiders(): import("./Spider.js").Spider[];
|
|
16
|
-
}
|
|
17
|
-
//# sourceMappingURL=ScrapyProjectAnalyzer.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ScrapyProjectAnalyzer.d.ts","sourceRoot":"","sources":["../../../../src/lib/projects/scrapy/ScrapyProjectAnalyzer.ts"],"names":[],"mappings":"AAGA,OAAO,YAAY,MAAM,cAAc,CAAC;AAKxC,qBAAa,qBAAqB;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,YAAY,CAAS;IACpC,QAAQ,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,cAAc,EAAE,MAAM,EAAE,CAAA;KAAE,GAAG,IAAI,CAAQ;gBAE3D,QAAQ,EAAE,MAAM;IAM5B,MAAM,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM;IAI9B,IAAI;IAIV,aAAa;IAaP,YAAY;IAuBlB,OAAO;IAIP,mBAAmB;CAsBtB"}
|
|
@@ -1,87 +0,0 @@
|
|
|
1
|
-
import { existsSync, readdirSync } from 'node:fs';
|
|
2
|
-
import { join, resolve } from 'node:path';
|
|
3
|
-
import ConfigParser from 'configparser';
|
|
4
|
-
import inquirer from 'inquirer';
|
|
5
|
-
import { SpiderFileAnalyzer } from './SpiderFileAnalyzer.js';
|
|
6
|
-
export class ScrapyProjectAnalyzer {
|
|
7
|
-
constructor(pathname) {
|
|
8
|
-
Object.defineProperty(this, "pathname", {
|
|
9
|
-
enumerable: true,
|
|
10
|
-
configurable: true,
|
|
11
|
-
writable: true,
|
|
12
|
-
value: void 0
|
|
13
|
-
});
|
|
14
|
-
Object.defineProperty(this, "configuration", {
|
|
15
|
-
enumerable: true,
|
|
16
|
-
configurable: true,
|
|
17
|
-
writable: true,
|
|
18
|
-
value: null
|
|
19
|
-
});
|
|
20
|
-
Object.defineProperty(this, "settings", {
|
|
21
|
-
enumerable: true,
|
|
22
|
-
configurable: true,
|
|
23
|
-
writable: true,
|
|
24
|
-
value: null
|
|
25
|
-
});
|
|
26
|
-
this.pathname = pathname;
|
|
27
|
-
this.settings = null;
|
|
28
|
-
this.loadScrapyCfg();
|
|
29
|
-
}
|
|
30
|
-
static isApplicable(pathname) {
|
|
31
|
-
return existsSync(join(pathname, 'scrapy.cfg'));
|
|
32
|
-
}
|
|
33
|
-
async init() {
|
|
34
|
-
await this.loadSettings();
|
|
35
|
-
}
|
|
36
|
-
loadScrapyCfg() {
|
|
37
|
-
const config = new ConfigParser();
|
|
38
|
-
const scrapyCfgPath = resolve(join(this.pathname, 'scrapy.cfg'));
|
|
39
|
-
if (!existsSync(scrapyCfgPath)) {
|
|
40
|
-
throw new Error(`scrapy.cfg not found in "${scrapyCfgPath}".
|
|
41
|
-
Are you sure there is a Scrapy project there?`);
|
|
42
|
-
}
|
|
43
|
-
config.read(scrapyCfgPath);
|
|
44
|
-
this.configuration = config;
|
|
45
|
-
}
|
|
46
|
-
async loadSettings() {
|
|
47
|
-
const assumedBotName = this.configuration.get('settings', 'default').split('.')[0];
|
|
48
|
-
const settings = await inquirer.prompt([
|
|
49
|
-
{
|
|
50
|
-
type: 'input',
|
|
51
|
-
name: 'BOT_NAME',
|
|
52
|
-
message: 'Enter the Scrapy BOT_NAME (see settings.py):',
|
|
53
|
-
default: assumedBotName,
|
|
54
|
-
},
|
|
55
|
-
{
|
|
56
|
-
type: 'input',
|
|
57
|
-
name: 'SPIDER_MODULES',
|
|
58
|
-
message: 'What folder are the Scrapy spider modules stored in? (see SPIDER_MODULES in settings.py):',
|
|
59
|
-
default: [`${assumedBotName}.spiders`],
|
|
60
|
-
},
|
|
61
|
-
]);
|
|
62
|
-
if (typeof settings.SPIDER_MODULES === 'string')
|
|
63
|
-
settings.SPIDER_MODULES = [settings.SPIDER_MODULES];
|
|
64
|
-
this.settings = settings;
|
|
65
|
-
}
|
|
66
|
-
getName() {
|
|
67
|
-
return this.settings?.BOT_NAME;
|
|
68
|
-
}
|
|
69
|
-
getAvailableSpiders() {
|
|
70
|
-
const spiderPaths = this.settings?.SPIDER_MODULES;
|
|
71
|
-
if (!spiderPaths) {
|
|
72
|
-
throw new Error('SPIDER_MODULES path not found in settings.');
|
|
73
|
-
}
|
|
74
|
-
const spiders = [];
|
|
75
|
-
for (const spiderPath of spiderPaths) {
|
|
76
|
-
const spidersDir = join(this.pathname, spiderPath.replaceAll('.', '/'));
|
|
77
|
-
const files = readdirSync(spidersDir, { withFileTypes: true });
|
|
78
|
-
for (const file of files) {
|
|
79
|
-
if (file.isFile() && file.name.endsWith('.py') && file.name !== '__init__.py') {
|
|
80
|
-
spiders.push(...(new SpiderFileAnalyzer(join(spidersDir, file.name)).getSpiders()));
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
return spiders;
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
//# sourceMappingURL=ScrapyProjectAnalyzer.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ScrapyProjectAnalyzer.js","sourceRoot":"","sources":["../../../../src/lib/projects/scrapy/ScrapyProjectAnalyzer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAClD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,YAAY,MAAM,cAAc,CAAC;AACxC,OAAO,QAAQ,MAAM,UAAU,CAAC;AAEhC,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAE7D,MAAM,OAAO,qBAAqB;IAK9B,YAAY,QAAgB;QAJ5B;;;;;WAAiB;QACjB;;;;mBAA8B,IAAK;WAAC;QACpC;;;;mBAAkE,IAAI;WAAC;QAGnE,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,aAAa,EAAE,CAAC;IACzB,CAAC;IAED,MAAM,CAAC,YAAY,CAAC,QAAgB;QAChC,OAAO,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,IAAI;QACN,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;IAC9B,CAAC;IAED,aAAa;QACT,MAAM,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;QAClC,MAAM,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC;QAEjE,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,4BAA4B,aAAa;8CACvB,CAAC,CAAC;QACxC,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC3B,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,YAAY;QACd,MAAM,cAAc,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,UAAU,EAAE,SAAS,CAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAEpF,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC;YACnC;gBACI,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,UAAU;gBAChB,OAAO,EAAE,8CAA8C;gBACvD,OAAO,EAAE,cAAc;aAC1B;YACD;gBACI,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,gBAAgB;gBACtB,OAAO,EAAE,2FAA2F;gBACpG,OAAO,EAAE,CAAC,GAAG,cAAc,UAAU,CAAC;aACzC;SACJ,CAAC,CAAC;QAEH,IAAI,OAAO,QAAQ,CAAC,cAAc,KAAK,QAAQ;YAAE,QAAQ,CAAC,cAAc,GAAG,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;QAErG,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC7B,CAAC;IAED,OAAO;QACH,OAAO,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC;IACnC,CAAC;IAED,mBAAmB;QACf,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,EAAE,cAAc,CAAC;QAElD,IAAI,CAAC,WAAW,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAClE,CAAC;QAED,MAAM,OAAO,GAAG,EAAE,CAAC;QAEnB,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;YACnC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;YAExE,MAAM,KAAK,GAAG,WAAW,CAAC,UAAU,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACvB,IAAI,IAAI,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;oBAC5E,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,kBAAkB,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;gBACxF,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;CACJ"}
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
export declare class Spider {
|
|
2
|
-
name?: string;
|
|
3
|
-
class_name: string;
|
|
4
|
-
start_urls?: string;
|
|
5
|
-
pathname: string;
|
|
6
|
-
constructor(data: SpiderData);
|
|
7
|
-
}
|
|
8
|
-
export interface SpiderData {
|
|
9
|
-
name?: string;
|
|
10
|
-
class_name: string;
|
|
11
|
-
start_urls?: string;
|
|
12
|
-
pathname: string;
|
|
13
|
-
}
|
|
14
|
-
//# sourceMappingURL=Spider.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"Spider.d.ts","sourceRoot":"","sources":["../../../../src/lib/projects/scrapy/Spider.ts"],"names":[],"mappings":"AAAA,qBAAa,MAAM;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;gBAEL,IAAI,EAAE,UAAU;CAM/B;AAED,MAAM,WAAW,UAAU;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;CACpB"}
|
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
export class Spider {
|
|
2
|
-
constructor(data) {
|
|
3
|
-
Object.defineProperty(this, "name", {
|
|
4
|
-
enumerable: true,
|
|
5
|
-
configurable: true,
|
|
6
|
-
writable: true,
|
|
7
|
-
value: void 0
|
|
8
|
-
});
|
|
9
|
-
Object.defineProperty(this, "class_name", {
|
|
10
|
-
enumerable: true,
|
|
11
|
-
configurable: true,
|
|
12
|
-
writable: true,
|
|
13
|
-
value: void 0
|
|
14
|
-
});
|
|
15
|
-
Object.defineProperty(this, "start_urls", {
|
|
16
|
-
enumerable: true,
|
|
17
|
-
configurable: true,
|
|
18
|
-
writable: true,
|
|
19
|
-
value: void 0
|
|
20
|
-
});
|
|
21
|
-
Object.defineProperty(this, "pathname", {
|
|
22
|
-
enumerable: true,
|
|
23
|
-
configurable: true,
|
|
24
|
-
writable: true,
|
|
25
|
-
value: void 0
|
|
26
|
-
});
|
|
27
|
-
this.name = data.name;
|
|
28
|
-
this.class_name = data.class_name;
|
|
29
|
-
this.start_urls = data.start_urls;
|
|
30
|
-
this.pathname = data.pathname;
|
|
31
|
-
}
|
|
32
|
-
}
|
|
33
|
-
//# sourceMappingURL=Spider.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"Spider.js","sourceRoot":"","sources":["../../../../src/lib/projects/scrapy/Spider.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,MAAM;IAMf,YAAY,IAAgB;QAL5B;;;;;WAAc;QACd;;;;;WAAmB;QACnB;;;;;WAAoB;QACpB;;;;;WAAiB;QAGb,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACtB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAClC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAClC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;IAClC,CAAC;CACJ"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"SpiderFileAnalyzer.d.ts","sourceRoot":"","sources":["../../../../src/lib/projects/scrapy/SpiderFileAnalyzer.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,qBAAa,kBAAkB;IAC3B,QAAQ,EAAE,MAAM,CAAC;gBAEL,QAAQ,EAAE,MAAM;IAI5B,UAAU;CAcb"}
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
import { readFileSync } from 'node:fs';
|
|
2
|
-
import { Spider } from './Spider.js';
|
|
3
|
-
export class SpiderFileAnalyzer {
|
|
4
|
-
constructor(pathname) {
|
|
5
|
-
Object.defineProperty(this, "pathname", {
|
|
6
|
-
enumerable: true,
|
|
7
|
-
configurable: true,
|
|
8
|
-
writable: true,
|
|
9
|
-
value: void 0
|
|
10
|
-
});
|
|
11
|
-
this.pathname = pathname;
|
|
12
|
-
}
|
|
13
|
-
getSpiders() {
|
|
14
|
-
const file = readFileSync(this.pathname, 'utf8');
|
|
15
|
-
const regex = /class\s+(\w+)/g;
|
|
16
|
-
const spiders = [];
|
|
17
|
-
let match = regex.exec(file);
|
|
18
|
-
while (match) {
|
|
19
|
-
spiders.push(new Spider({ class_name: match[1], pathname: this.pathname }));
|
|
20
|
-
match = regex.exec(file);
|
|
21
|
-
}
|
|
22
|
-
return spiders;
|
|
23
|
-
}
|
|
24
|
-
}
|
|
25
|
-
//# sourceMappingURL=SpiderFileAnalyzer.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"SpiderFileAnalyzer.js","sourceRoot":"","sources":["../../../../src/lib/projects/scrapy/SpiderFileAnalyzer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAEvC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,MAAM,OAAO,kBAAkB;IAG3B,YAAY,QAAgB;QAF5B;;;;;WAAiB;QAGb,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC7B,CAAC;IAED,UAAU;QACN,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAEjD,MAAM,KAAK,GAAG,gBAAgB,CAAC;QAC/B,MAAM,OAAO,GAAG,EAAE,CAAC;QAEnB,IAAI,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,OAAO,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;YAC5E,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;CACJ"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"wrapScrapyProject.d.ts","sourceRoot":"","sources":["../../../../src/lib/projects/scrapy/wrapScrapyProject.ts"],"names":[],"mappings":"AA2DA,wBAAsB,iBAAiB,CAAC,EAAE,WAAW,EAAE,EAAE;IAAE,WAAW,CAAC,EAAE,MAAM,CAAC;CAAE,iBAgFjF"}
|