bananareporter 0.2.0 → 0.3.0

package/README.md

@@ -26,8 +26,8 @@
 - **Multiple output formats**: output file can be a JSON, JSONL or CSV (via `--format`) _more coming soon_
 - **Configurable**: data to be imported can be filtered using the configuration file and each option can be overridden per source
 - **Sources Supported**
-  - GitLab: commits
-  - GitHub: commits
+  - [GitLab](./docs/sources/gitlab.md): commits
+  - [GitHub](./docs/sources/github.md): commits
   - todo.txt: tasks
   - **[Request a new source](https://github.com/nya1/bananareporter/issues/new?assignees=&labels=enhancement&template=new-source-request.md&title=)**
 
@@ -67,14 +67,15 @@ sources:
 bananareporter --from 2023-01-01 --to 2023-03-01 -c bananareporter.yaml
 ```
 
-In the current directory you will find the output as `bananareporter_$FROM__$TO.json`, can be changed with `--out`
+In the current directory you will find the output as `bananareporter_$FROM__$TO.json` (can be changed with `--out`)
 
 Example of output (json) with gitlab and github sources:
 
 ```json
 [
   {
-    "date": "2022-07-13T07:51:21.730Z",
+    "id": "c12ba180bfecf45fcdcc40d6104d1f1b7ad409dc",
+    "date": "2023-01-13T07:51:21.730Z",
     "username": "johndoe",
     "description": "chore: update changelog and swagger branch:work git:aa33b04",
     "projectId": "3318214",
@@ -82,8 +83,9 @@ Example of output (json) with gitlab and github sources:
     "type": "gitlab"
   },
   {
-    "date": "2022-07-14T10:50:10.230Z",
-    "username": "johndoe2",
+    "id": "6e1b66a1dea89e957d8c44943f942be4874c0641",
+    "date": "2023-01-14T10:50:10.230Z",
+    "username": "johndoe",
     "description": "refactor: compare date function branch:work git:ia1f241",
     "projectId": "928544",
     "projectName": "awesome-backend",
@@ -100,7 +102,7 @@ $ npm install -g bananareporter
 $ bananareporter COMMAND
 running command...
 $ bananareporter (--version)
-bananareporter/0.2.0 linux-x64 node-v18.14.2
+bananareporter/0.3.0 linux-x64 node-v18.15.0
 $ bananareporter --help [COMMAND]
 USAGE
   $ bananareporter COMMAND
@@ -130,7 +132,7 @@ DESCRIPTION
   Display help for bananareporter.
 ```
 
-_See code: [@oclif/plugin-help](https://github.com/oclif/plugin-help/blob/v5.2.7/src/commands/help.ts)_
+_See code: [@oclif/plugin-help](https://github.com/oclif/plugin-help/blob/v5.2.8/src/commands/help.ts)_
 
 ## `bananareporter run`
 
@@ -143,7 +145,7 @@ USAGE
 
 FLAGS
   -c, --config=<value>  config file location, by default ~/.config/bananareporter/config.yaml
-  -o, --out=<value>     (required) [default: ./bananareporter_$FROM__$TO.json] file path to save the output
+  -o, --out=<value>     (required) [default: ./bananareporter_$FROM__$TO.json] file path or directory to save the output
   --delay=<value>       [default: 300] global delay in millisecons between http requests
   --format=<option>     (required) [default: json] output file format
                         <options: json|jsonl|csv>
@@ -159,5 +161,5 @@ EXAMPLES
   report with 138 entries saved to ./bananareporter.json
 ```
 
-_See code: [dist/commands/run/index.ts](https://github.com/nya1/bananareporter/blob/v0.2.0/dist/commands/run/index.ts)_
+_See code: [dist/commands/run/index.ts](https://github.com/nya1/bananareporter/blob/v0.3.0/dist/commands/run/index.ts)_
 <!-- commandsstop -->

package/dist/commands/run/index.d.ts

@@ -14,4 +14,5 @@ export default class Run extends Command {
         'disable-welcome-emoji': import("@oclif/core/lib/interfaces").BooleanFlag<boolean>;
     };
     run(): Promise<void>;
+    catch(error: Error): Promise<void>;
 }

package/dist/commands/run/index.js

@@ -13,6 +13,8 @@ const lodash_1 = require("lodash");
 const logger_1 = require("../../util/logger");
 const github_1 = require("../../integrations/github");
 const todotxt_1 = require("../../integrations/todotxt");
+const zod_1 = require("zod");
+const node_path_1 = require("node:path");
 class Run extends core_1.Command {
     static dateStringValidation(input) {
         if (!dayjs(input).isValid()) {
@@ -97,23 +99,47 @@ class Run extends core_1.Command {
         }
         const fileExt = `.${validatedConfig.format}`;
         let outFile = validatedConfig.out;
+        logger_1.logger.debug(`Run.flags.out.default ${Run.flags.out.default}`);
+        logger_1.logger.debug(`fileExt ${fileExt}`);
+        logger_1.logger.debug(`outFile ${outFile}`);
+        // if --out was provided and it's a directory
+        // use the default file name
+        try {
+            if ((0, node_fs_1.statSync)(outFile).isDirectory()) {
+                outFile = (0, node_path_1.resolve)(outFile, Run.flags.out.default);
+                logger_1.logger.debug(`outFile is a directory, updated to ${outFile}`);
+            }
+        }
+        catch (error) {
+            logger_1.logger.debug('statSync error', error);
+        }
         // add file extension
         if (!outFile.endsWith(fileExt)) {
             if (outFile === Run.flags.out.default) {
                 outFile = outFile
-                    .replace(`.${Run.flags.format.default}`, fileExt)
-                    .replace('$FROM', validatedConfig.from)
-                    .replace('$TO', validatedConfig.to);
+                    .replace(`.${Run.flags.format.default}`, fileExt);
             }
             else {
                 outFile += fileExt;
             }
         }
+        if (outFile.includes('$FROM') || outFile.includes('$TO')) {
+            outFile = outFile
+                .replace('$FROM', validatedConfig.from)
+                .replace('$TO', validatedConfig.to);
+        }
         logger_1.logger.debug(`run saving to ${outFile} with file content of size ${outputStr.length}`);
         (0, node_fs_1.writeFileSync)(outFile, outputStr);
         this.log(`report with ${reportList.length} entries saved to ${outFile}`);
         // this.log(`run! ${flags.config} ${JSON.stringify(configContent)}`)
     }
+    async catch(error) {
+        if (error instanceof zod_1.z.ZodError) {
+            const errMsg = error.issues.map(i => `${i.message}: ${i.path.join(', ')}`).join('\n');
+            throw new TypeError(`validation failed, ${errMsg}`);
+        }
+        throw error;
+    }
 }
 exports.default = Run;
 Run.description = 'Run report';
@@ -143,7 +169,7 @@ Run.flags = {
     out: core_1.Flags.file({
         char: 'o',
         aliases: ['output'],
-        description: 'file path to save the output',
+        description: 'file path or directory to save the output',
         required: true,
         default: './bananareporter_$FROM__$TO.json',
     }),

package/dist/core/index.d.ts

@@ -24,15 +24,11 @@ export declare const BananaConfig: z.ZodEffects<z.ZodEffects<z.ZodObject<{
        */
     sources: z.ZodArray<z.ZodObject<{
         type: z.ZodEnum<["gitlab", "github", "todo.txt"]>;
-    }, "passthrough", z.ZodTypeAny, {
-        type: "gitlab" | "github" | "todo.txt";
-    } & {
-        [k: string]: unknown;
-    }, {
-        type: "gitlab" | "github" | "todo.txt";
-    } & {
-        [k: string]: unknown;
-    }>, "atleastone">;
+    }, "passthrough", z.ZodTypeAny, z.objectOutputType<{
+        type: z.ZodEnum<["gitlab", "github", "todo.txt"]>;
+    }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+        type: z.ZodEnum<["gitlab", "github", "todo.txt"]>;
+    }, z.ZodTypeAny, "passthrough">>, "atleastone">;
 }, "strip", z.ZodTypeAny, {
     format: "json" | "csv" | "jsonl";
     from: string;
@@ -41,32 +37,24 @@ export declare const BananaConfig: z.ZodEffects<z.ZodEffects<z.ZodObject<{
     to: string;
     configFileLocation: string;
     delay: number;
-    sources: [{
-        type: "gitlab" | "github" | "todo.txt";
-    } & {
-        [k: string]: unknown;
-    }, ...({
-        type: "gitlab" | "github" | "todo.txt";
-    } & {
-        [k: string]: unknown;
-    })[]];
+    sources: [z.objectOutputType<{
+        type: z.ZodEnum<["gitlab", "github", "todo.txt"]>;
+    }, z.ZodTypeAny, "passthrough">, ...z.objectOutputType<{
+        type: z.ZodEnum<["gitlab", "github", "todo.txt"]>;
+    }, z.ZodTypeAny, "passthrough">[]];
 }, {
-    includeRawObject?: boolean | undefined;
-    delay?: number | undefined;
     format: "json" | "csv" | "jsonl";
     from: string;
     out: string;
     to: string;
     configFileLocation: string;
-    sources: [{
-        type: "gitlab" | "github" | "todo.txt";
-    } & {
-        [k: string]: unknown;
-    }, ...({
-        type: "gitlab" | "github" | "todo.txt";
-    } & {
-        [k: string]: unknown;
-    })[]];
+    sources: [z.objectInputType<{
+        type: z.ZodEnum<["gitlab", "github", "todo.txt"]>;
+    }, z.ZodTypeAny, "passthrough">, ...z.objectInputType<{
+        type: z.ZodEnum<["gitlab", "github", "todo.txt"]>;
+    }, z.ZodTypeAny, "passthrough">[]];
+    includeRawObject?: boolean | undefined;
+    delay?: number | undefined;
 }>, {
     format: "json" | "csv" | "jsonl";
     from: string;
@@ -75,32 +63,24 @@ export declare const BananaConfig: z.ZodEffects<z.ZodEffects<z.ZodObject<{
     to: string;
     configFileLocation: string;
     delay: number;
-    sources: [{
-        type: "gitlab" | "github" | "todo.txt";
-    } & {
-        [k: string]: unknown;
-    }, ...({
-        type: "gitlab" | "github" | "todo.txt";
-    } & {
-        [k: string]: unknown;
-    })[]];
+    sources: [z.objectOutputType<{
+        type: z.ZodEnum<["gitlab", "github", "todo.txt"]>;
+    }, z.ZodTypeAny, "passthrough">, ...z.objectOutputType<{
+        type: z.ZodEnum<["gitlab", "github", "todo.txt"]>;
+    }, z.ZodTypeAny, "passthrough">[]];
 }, {
-    includeRawObject?: boolean | undefined;
-    delay?: number | undefined;
     format: "json" | "csv" | "jsonl";
     from: string;
     out: string;
     to: string;
     configFileLocation: string;
-    sources: [{
-        type: "gitlab" | "github" | "todo.txt";
-    } & {
-        [k: string]: unknown;
-    }, ...({
-        type: "gitlab" | "github" | "todo.txt";
-    } & {
-        [k: string]: unknown;
-    })[]];
+    sources: [z.objectInputType<{
+        type: z.ZodEnum<["gitlab", "github", "todo.txt"]>;
+    }, z.ZodTypeAny, "passthrough">, ...z.objectInputType<{
+        type: z.ZodEnum<["gitlab", "github", "todo.txt"]>;
+    }, z.ZodTypeAny, "passthrough">[]];
+    includeRawObject?: boolean | undefined;
+    delay?: number | undefined;
 }>, {
     format: "json" | "csv" | "jsonl";
     from: string;
@@ -109,32 +89,24 @@ export declare const BananaConfig: z.ZodEffects<z.ZodEffects<z.ZodObject<{
     to: string;
     configFileLocation: string;
     delay: number;
-    sources: [{
-        type: "gitlab" | "github" | "todo.txt";
-    } & {
-        [k: string]: unknown;
-    }, ...({
-        type: "gitlab" | "github" | "todo.txt";
-    } & {
-        [k: string]: unknown;
-    })[]];
+    sources: [z.objectOutputType<{
+        type: z.ZodEnum<["gitlab", "github", "todo.txt"]>;
+    }, z.ZodTypeAny, "passthrough">, ...z.objectOutputType<{
+        type: z.ZodEnum<["gitlab", "github", "todo.txt"]>;
+    }, z.ZodTypeAny, "passthrough">[]];
 }, {
-    includeRawObject?: boolean | undefined;
-    delay?: number | undefined;
     format: "json" | "csv" | "jsonl";
     from: string;
     out: string;
     to: string;
     configFileLocation: string;
-    sources: [{
-        type: "gitlab" | "github" | "todo.txt";
-    } & {
-        [k: string]: unknown;
-    }, ...({
-        type: "gitlab" | "github" | "todo.txt";
-    } & {
-        [k: string]: unknown;
-    })[]];
+    sources: [z.objectInputType<{
+        type: z.ZodEnum<["gitlab", "github", "todo.txt"]>;
+    }, z.ZodTypeAny, "passthrough">, ...z.objectInputType<{
+        type: z.ZodEnum<["gitlab", "github", "todo.txt"]>;
+    }, z.ZodTypeAny, "passthrough">[]];
+    includeRawObject?: boolean | undefined;
+    delay?: number | undefined;
 }>;
 export type BananaConfig = z.infer<typeof BananaConfig>;
 export declare function loadAndValidateConfig(input: unknown): BananaConfig;

package/dist/integrations/base.d.ts

@@ -3,13 +3,18 @@ import { BananaConfig } from '../core';
 export declare const SourceType: z.ZodEnum<["gitlab", "github", "todo.txt"]>;
 export type SourceType = z.infer<typeof SourceType>;
 export interface CommonBananaReporterObj {
+    id: string;
     date: string;
     description: string;
     username?: string;
     projectId?: string;
     projectName?: string;
     type: SourceType;
-    __raw: any;
+    /**
+     * the raw source object,
+     * included only when --include-raw-object is provided
+     */
+    __raw?: any;
 }
 export declare abstract class IntegrationBase {
     static type: SourceType;

package/dist/integrations/github.d.ts

@@ -167,9 +167,10 @@ export declare const GithubConfig: z.ZodObject<{
     to: z.ZodOptional<z.ZodString>;
     delay: z.ZodOptional<z.ZodNumber>;
 }, "strip", z.ZodTypeAny, {
-    from?: string | undefined;
-    to?: string | undefined;
-    delay?: number | undefined;
+    protocol: "http" | "https";
+    committerUsername: string;
+    domain: string;
+    apiVersion: string;
     token?: string | undefined;
     filters?: [{
         on: string;
@@ -178,15 +179,11 @@ export declare const GithubConfig: z.ZodObject<{
         on: string;
         regex: string;
     }[]] | undefined;
-    protocol: "http" | "https";
-    committerUsername: string;
-    domain: string;
-    apiVersion: string;
-}, {
     from?: string | undefined;
-    protocol?: "http" | "https" | undefined;
     to?: string | undefined;
     delay?: number | undefined;
+}, {
+    committerUsername: string;
     token?: string | undefined;
     filters?: [{
         on: string;
@@ -197,7 +194,10 @@ export declare const GithubConfig: z.ZodObject<{
     }[]] | undefined;
     domain?: string | undefined;
     apiVersion?: string | undefined;
-    committerUsername: string;
+    protocol?: "http" | "https" | undefined;
+    from?: string | undefined;
+    to?: string | undefined;
+    delay?: number | undefined;
 }>;
 export declare class GithubIntegration extends IntegrationBase {
     static type: "github";

package/dist/integrations/github.js

@@ -65,6 +65,7 @@ class GithubIntegration extends base_1.IntegrationBase {
     async fetchData() {
         let page = 1;
         let commitList = [];
+        const commitShaList = new Set();
         while (page > 0) {
             logger_1.logger.debug(`github integration working on ${page}`);
             /* eslint-disable no-await-in-loop */
@@ -79,7 +80,12 @@ class GithubIntegration extends base_1.IntegrationBase {
                 break;
             }
             // eslint-disable-next-line unicorn/prefer-spread
-            commitList = commitList.concat(commits.items);
+            commitList = commitList.concat(
+            // remove duplicate items
+            commits.items.filter(c => !commitShaList.has(c.sha)));
+            for (const c of commits.items) {
+                commitShaList.add(c.sha);
+            }
             logger_1.logger.debug(`github integration commitList ${commitList.length} (added ${commits.items.length} commits)`);
             page += 1;
             await (0, common_1.delay)(this.delayToUse);
@@ -126,6 +132,7 @@ class GithubIntegration extends base_1.IntegrationBase {
         const projectName = rawData.repository.name;
         const description = `${rawData.commit.message} git:${rawData.sha.slice(0, 7)}`;
         return {
+            id: rawData.sha,
             date: rawData.commit.committer.date,
             username: rawData.commit.committer.name,
             description,

package/dist/integrations/gitlab.d.ts

@@ -54,9 +54,12 @@ export declare const GitlabConfig: z.ZodObject<{
     to: z.ZodOptional<z.ZodString>;
     delay: z.ZodOptional<z.ZodNumber>;
 }, "strip", z.ZodTypeAny, {
-    from?: string | undefined;
-    to?: string | undefined;
-    delay?: number | undefined;
+    protocol: "http" | "https";
+    committerUsername: string;
+    token: string;
+    domain: string;
+    apiVersion: string;
+    apiBasePath: string;
     userId?: number | undefined;
     filters?: [{
         on: string;
@@ -65,17 +68,12 @@ export declare const GitlabConfig: z.ZodObject<{
         on: string;
         regex: string;
     }[]] | undefined;
-    protocol: "http" | "https";
-    committerUsername: string;
-    token: string;
-    domain: string;
-    apiVersion: string;
-    apiBasePath: string;
-}, {
     from?: string | undefined;
-    protocol?: "http" | "https" | undefined;
     to?: string | undefined;
     delay?: number | undefined;
+}, {
+    committerUsername: string;
+    token: string;
     userId?: number | undefined;
     filters?: [{
         on: string;
@@ -86,9 +84,11 @@ export declare const GitlabConfig: z.ZodObject<{
     }[]] | undefined;
     domain?: string | undefined;
     apiVersion?: string | undefined;
+    protocol?: "http" | "https" | undefined;
     apiBasePath?: string | undefined;
-    committerUsername: string;
-    token: string;
+    from?: string | undefined;
+    to?: string | undefined;
+    delay?: number | undefined;
 }>;
 export declare class GitlabIntegration extends IntegrationBase {
     static type: "gitlab";

package/dist/integrations/gitlab.js

@@ -88,6 +88,7 @@ class GitlabIntegration extends base_1.IntegrationBase {
         let page = 1;
         let eventList = [];
         const projectIds = new Set();
+        const commitShaList = new Set();
         while (page > 0) {
             logger_1.logger.debug(`gitlab integration working on ${page}`);
             /* eslint-disable no-await-in-loop */
@@ -101,11 +102,14 @@ class GitlabIntegration extends base_1.IntegrationBase {
                 logger_1.logger.debug('gitlab integration no more events to process');
                 break;
             }
+            // eslint-disable-next-line unicorn/prefer-spread
+            eventList = eventList.concat(
+            // remove duplicated commits
+            events.filter(e => !commitShaList.has(e.push_data.commit_to)));
             for (const e of events) {
+                commitShaList.add(e.push_data.commit_to);
                 projectIds.add(e.project_id);
             }
-            // eslint-disable-next-line unicorn/prefer-spread
-            eventList = eventList.concat(events);
             logger_1.logger.debug(`gitlab integration eventList ${eventList.length} (added ${events.length} events)`);
             page += 1;
             await (0, common_1.delay)(this.delayToUse);
@@ -240,6 +244,7 @@ class GitlabIntegration extends base_1.IntegrationBase {
         const projectName = projectDetails === null || projectDetails === void 0 ? void 0 : projectDetails.path;
         const description = `${rawData.push_data.commit_title} branch:${rawData.push_data.ref} git:${rawData.push_data.commit_to.slice(0, 7)}`;
         return {
+            id: `${String(rawData.push_data.commit_to)}`,
             date: rawData.created_at,
             username: rawData.author_username,
             description,

package/dist/integrations/todotxt.d.ts

@@ -18,8 +18,7 @@ export declare const TodoTxtConfig: z.ZodObject<{
     from: z.ZodOptional<z.ZodString>;
     to: z.ZodOptional<z.ZodString>;
 }, "strip", z.ZodTypeAny, {
-    from?: string | undefined;
-    to?: string | undefined;
+    file: string;
     filters?: [{
         on: string;
         regex: string;
@@ -27,11 +26,10 @@ export declare const TodoTxtConfig: z.ZodObject<{
         on: string;
         regex: string;
     }[]] | undefined;
-    file: string;
-}, {
-    file?: string | undefined;
     from?: string | undefined;
     to?: string | undefined;
+}, {
+    file?: string | undefined;
     filters?: [{
         on: string;
         regex: string;
@@ -39,6 +37,8 @@ export declare const TodoTxtConfig: z.ZodObject<{
         on: string;
         regex: string;
     }[]] | undefined;
+    from?: string | undefined;
+    to?: string | undefined;
 }>;
 export declare class TodoTxtIntegration extends IntegrationBase {
     static type: "todo.txt";
@@ -47,5 +47,5 @@ export declare class TodoTxtIntegration extends IntegrationBase {
     private bananaReporterConfig;
     constructor(_rawConfig: unknown, bananaReporterConfig: BananaConfig);
     fetchData(): Promise<CommonBananaReporterObj[]>;
-    toBananaReporterObj(rawData: Task): CommonBananaReporterObj;
+    toBananaReporterObj(rawData: Task, index?: number): CommonBananaReporterObj;
 }

package/dist/integrations/todotxt.js

@@ -73,14 +73,15 @@ class TodoTxtIntegration extends base_1.IntegrationBase {
                 return regex.test(value);
             });
         }
-        const formattedData = filteredTasks.map(e => this.toBananaReporterObj(e));
+        const formattedData = filteredTasks.map((e, i) => this.toBananaReporterObj(e, i));
         return formattedData;
     }
-    toBananaReporterObj(rawData) {
+    toBananaReporterObj(rawData, index) {
         var _a;
         const projectName = (_a = rawData.projects) === null || _a === void 0 ? void 0 : _a.map(p => p.replace('+', '')).join(',');
         const description = `${rawData.description}`;
         return {
+            id: `${index}`,
             date: rawData.completion || rawData.creation || '',
             description,
             projectName,

package/oclif.manifest.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.2.0",
+  "version": "0.3.0",
   "commands": {
     "run": {
       "id": "run",
@@ -44,7 +44,7 @@
           "name": "out",
           "type": "option",
           "char": "o",
-          "description": "file path to save the output",
+          "description": "file path or directory to save the output",
           "required": true,
           "multiple": false,
           "default": "./bananareporter_$FROM__$TO.json",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bananareporter",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Easily generate a report from multiple sources",
   "author": "nya1",
   "bin": {
@@ -27,14 +27,14 @@
     "object-path": "^0.11.8",
     "papaparse": "^5.4.0",
     "winston": "^3.8.2",
-    "zod": "^3.21.0"
+    "zod": "^3.21.4"
   },
   "devDependencies": {
-    "@oclif/test": "^2.3.8",
+    "@oclif/test": "^2.3.10",
     "@types/chai": "^4",
     "@types/js-yaml": "^4.0.5",
     "@types/mocha": "^9.1.1",
-    "@types/node": "^18.14.6",
+    "@types/node": "^18.15.3",
     "@types/node-fetch": "^2.6.2",
     "@types/object-path": "^0.11.1",
     "@types/papaparse": "^5.3.7",