luca 1.0.1 → 1.1.1

package/dist/config/consts.d.ts

@@ -1,2 +1,2 @@
-export declare const getConfigPath: () => string;
-export declare const getResultPath: () => string;
+export declare const CONFIG_PATH: string;
+export declare const LOG_PATH: string;

package/dist/config/consts.js

@@ -23,9 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getResultPath = exports.getConfigPath = void 0;
+exports.LOG_PATH = exports.CONFIG_PATH = void 0;
 const path = __importStar(require("path"));
-const getConfigPath = () => path.resolve('luca.config.js');
-exports.getConfigPath = getConfigPath;
-const getResultPath = () => path.resolve('luca.result.json');
-exports.getResultPath = getResultPath;
+exports.CONFIG_PATH = path.resolve('luca.config.js');
+exports.LOG_PATH = path.resolve('luca.log');

package/dist/config/default.d.ts

@@ -0,0 +1,8 @@
+export const delay: number;
+export const baseUrl: string;
+export function query(baseUrl: any, item: any): string;
+export function handler(response: any, item: any): {
+    item: any;
+    data: any;
+};
+export const items: string[];

package/dist/config/default.js

@@ -0,0 +1,15 @@
+"use strict";
+module.exports = {
+    delay: 500,
+    baseUrl: 'https://www.npmjs.com/package',
+    query: (baseUrl, item) => {
+        return `${baseUrl}/${item}`;
+    },
+    handler: (response, item) => {
+        return {
+            item,
+            data: response.data,
+        };
+    },
+    items: ['foo', 'bar'],
+};

package/dist/config/initConfig.js

@@ -38,24 +38,12 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.initConfig = void 0;
 const chalk_1 = __importDefault(require("chalk"));
 const fs = __importStar(require("fs/promises"));
+const path = __importStar(require("path"));
 const consts_1 = require("./consts");
 const initConfig = () => __awaiter(void 0, void 0, void 0, function* () {
     try {
-        const config = (0, consts_1.getConfigPath)();
-        yield fs.writeFile(config, `module.exports = {
-    delay: 500,
-    baseUrl: "https://www.npmjs.com/package",
-    query: (baseUrl, item) => {
-        return \`\${baseUrl}/\${item}\`;
-    },
-    handler: (response, item) => {
-        return {
-            status: response.status,
-        };
-    },
-    items: ['foo', 'bar'],
-}`);
-        console.log(chalk_1.default.green(`Configuration file was created:\n> ${config}`));
+        yield fs.copyFile(path.resolve(__dirname, `default.js`), consts_1.CONFIG_PATH);
+        console.log(chalk_1.default.green(`Configuration file was created:\n> ${consts_1.CONFIG_PATH}`));
     }
     catch (e) {
         console.error(e);

package/dist/config/openConfig.d.ts

@@ -0,0 +1,2 @@
+import { TConfig } from './types';
+export declare const openConfig: () => TConfig;

package/dist/config/{validateConfig.js → openConfig.js}

@@ -1,7 +1,14 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.validateConfig = void 0;
-const validateConfig = (config) => {
+exports.openConfig = void 0;
+const utils_1 = require("../utils");
+const consts_1 = require("./consts");
+const openConfig = () => {
+    if (!(0, utils_1.checkPath)(consts_1.CONFIG_PATH)) {
+        throw new Error(`Config not found`);
+    }
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const config = require(consts_1.CONFIG_PATH);
     const { baseUrl, items, query, handler } = config;
     if (items.length === 0) {
         throw new Error(`Specify config items, eg: items: ["foo", "bar"]`);
@@ -14,4 +21,4 @@ const validateConfig = (config) => {
     }
     return config;
 };
-exports.validateConfig = validateConfig;
+exports.openConfig = openConfig;

package/dist/config/runConfig.d.ts

@@ -1,4 +1,3 @@
 import { AxiosError } from 'axios';
-import { TConfig } from './types';
 export declare function isAxiosError(error: any): error is AxiosError;
-export declare const runConfig: (options?: TConfig) => Promise<void>;
+export declare const runConfig: () => Promise<void>;

package/dist/config/runConfig.js

@@ -41,7 +41,7 @@ const chalk_1 = __importDefault(require("chalk"));
 const fs = __importStar(require("fs/promises"));
 const utils_1 = require("../utils");
 const consts_1 = require("./consts");
-const validateConfig_1 = require("./validateConfig");
+const openConfig_1 = require("./openConfig");
 function isAxiosError(error) {
     if ('response' in error) {
         return true;
@@ -50,52 +50,65 @@ function isAxiosError(error) {
 }
 exports.isAxiosError = isAxiosError;
 const log = console.log;
-// TODO: error retry
-// TODO: run from last element
-// TODO: clear result
-// TODO: validat config - no config
-const runConfig = (options = require((0, consts_1.getConfigPath)())) => __awaiter(void 0, void 0, void 0, function* () {
-    const resultPath = (0, consts_1.getResultPath)();
+const ONE_MINUTE = 60 * 1000;
+const handleRequest = (options) => __awaiter(void 0, void 0, void 0, function* () {
+    const { url, num, total, sleepDelay = ONE_MINUTE, sleepCount = 0 } = options;
+    let response = null;
+    let isErroredRequest = false;
     try {
+        response = yield axios_1.default.get(url);
+    }
+    catch (e) {
+        isErroredRequest = true;
+        if (isAxiosError(e) && e.response) {
+            response = e.response;
+        }
+    }
+    finally {
+        log(chalk_1.default.blue(`${num}/${total}`), `GET ${chalk_1.default[isErroredRequest ? 'red' : 'green'](`(${response === null || response === void 0 ? void 0 : response.status})`)}:`, url);
+    }
+    // Handle: Too Many Requests
+    if (sleepCount > 3) {
+        throw new Error(`Exceeded ${sleepCount} requests`);
+    }
+    if (response && response.status === 429) {
+        log(chalk_1.default.grey(`sleep...`));
+        yield (0, utils_1.wait)(sleepDelay);
+        return handleRequest(Object.assign(Object.assign({}, options), { sleepDelay: sleepDelay + ONE_MINUTE, sleepCount: sleepCount + 1 }));
+    }
+    return response;
+});
+const writeToLog = (...strs) => __awaiter(void 0, void 0, void 0, function* () {
+    yield fs.appendFile(consts_1.LOG_PATH, '\n' + strs.join('\n'));
+});
+// TODO: run from last log element
+const runConfig = () => __awaiter(void 0, void 0, void 0, function* () {
+    try {
+        const config = (0, openConfig_1.openConfig)();
+        const { baseUrl, items, query } = config;
         log(chalk_1.default.blue('Luca is running...'));
-        const config = (0, validateConfig_1.validateConfig)(options);
-        const { baseUrl, items, query, handler } = config;
+        yield writeToLog('Start session', `time: ${new Date()}`);
         const total = items.length;
         let index = 0;
         for (const item of items) {
             const num = (index += 1);
-            const isFirst = num === 1;
-            const isLast = num === total;
             const url = query(baseUrl, item);
-            let response = null;
-            let isErroredRequest = false;
-            try {
-                response = yield axios_1.default.get(url);
-            }
-            catch (e) {
-                isErroredRequest = true;
-                if (isAxiosError(e) && e.response) {
-                    response = e.response;
-                }
-            }
-            finally {
-                log(chalk_1.default.blue(`${num}/${total}`), `GET ${chalk_1.default[isErroredRequest ? 'red' : 'green'](`(${response === null || response === void 0 ? void 0 : response.status})`)}:`, url);
-            }
             try {
-                const data = handler(response, item);
-                yield fs.appendFile(resultPath, `${isFirst ? '[' : ''}${JSON.stringify(data)}${isLast ? ']' : ','}`);
+                const response = yield handleRequest({ url, num, total });
+                yield writeToLog(`\nurl: ${url}`, `status: ${response === null || response === void 0 ? void 0 : response.status}`, `data: ${response ? JSON.stringify(config.handler(response, item)) : null}`);
             }
             catch (error) {
                 console.error('handler error', error);
             }
             finally {
-                yield (0, utils_1.delay)(options.delay);
+                yield (0, utils_1.wait)(config.delay);
             }
         }
-        log(chalk_1.default.green(`Finished:\n> ${resultPath}`));
+        log(chalk_1.default.green(`Finished:\n> ${consts_1.LOG_PATH}`));
+        writeToLog(`\nEnd session`, `time: ${new Date()}`, '');
     }
     catch (e) {
-        console.error(e);
+        log(chalk_1.default.red(e));
     }
 });
 exports.runConfig = runConfig;

package/dist/config/types.d.ts

@@ -6,3 +6,7 @@ export interface TConfig<T = unknown> {
     query: (baseUrl: string, item: T) => string;
     handler: (response: AxiosResponse | null, item: T) => any;
 }
+export interface THandledData<T = unknown> {
+    status: number | null;
+    data: T | null;
+}

package/dist/utils/common.d.ts

@@ -1,2 +1,2 @@
-export declare function isDir(path: string): boolean;
-export declare const delay: (ms: number) => Promise<unknown>;
+export declare function checkPath(path: string, isDir?: boolean): boolean;
+export declare const wait: (ms: number) => Promise<unknown>;

package/dist/utils/common.js

@@ -32,18 +32,21 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.delay = exports.isDir = void 0;
+exports.wait = exports.checkPath = void 0;
 const fs = __importStar(require("fs"));
-function isDir(path) {
+function checkPath(path, isDir = false) {
     try {
         const stat = fs.lstatSync(path);
-        return stat.isDirectory();
+        if (isDir) {
+            return stat.isDirectory();
+        }
+        return stat.isFile();
     }
     catch (_a) {
         // lstatSync throws an error if path doesn't exist
         return false;
     }
 }
-exports.isDir = isDir;
-const delay = (ms) => __awaiter(void 0, void 0, void 0, function* () { return new Promise((res) => setTimeout(res, ms)); });
-exports.delay = delay;
+exports.checkPath = checkPath;
+const wait = (ms) => __awaiter(void 0, void 0, void 0, function* () { return new Promise((res) => setTimeout(res, ms)); });
+exports.wait = wait;

package/package.json

@@ -16,7 +16,7 @@
         "dist"
     ],
     "license": "MIT",
-    "version": "1.0.1",
+    "version": "1.1.1",
     "scripts": {
         "build": "tsc",
         "dev": "tsc-watch ./src/dev.ts --outDir ./dist/ --onSuccess \"node ./dist/dev.js\" --esModuleInterop",

package/dist/config/validateConfig.d.ts

@@ -1,2 +0,0 @@
-import { TConfig } from './types';
-export declare const validateConfig: (config: TConfig) => TConfig;