@into-cps-association/libms 0.4.7 → 0.5.1

package/eslint.config.js

@@ -20,6 +20,11 @@ export default [
     files: ['src/**', 'test/**'],
   },
   prettier,
+  {
+    ignores: [
+      'test/update-config.js', // Add the file you want to ignore
+    ],
+  },
   {
     languageOptions: {
       globals: {

package/libms.yaml.sample

@@ -0,0 +1,12 @@
+port: '4001'
+mode: 'git'
+local-path: '..\..\files'
+log-level: 'debug'
+apollo-path: '/lib'
+graphql-playground: 'true'
+
+git-repos:
+  - user-1: 
+      repo-url: 'https://github.com/isomorphic-git/lightning-fs'
+  - user-2: 
+      repo-url: 'https://github.com/isomorphic-git/lightning-fs'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@into-cps-association/libms",
-  "version": "0.4.7",
+  "version": "0.5.1",
   "description": "microservices that handles request by fetching and returning the file-names and folders of given directory",
   "author": "phillip.boe.jensen@gmail.com",
   "contributors": [
@@ -24,14 +24,14 @@
     "syntax": "eslint . --fix",
     "pretest": "npx shx cp test/data/user2/tools/README.md ../../files/user2/tools/README.md",
     "posttest": "npx rimraf ../../files/user2/tools/README.md",
-    "test:all": "npx cross-env NODE_OPTIONS=--experimental-vm-modules LOCAL_PATH=test/data jest --testPathIgnorePatterns=cloudcmd --coverage",
-    "test:e2e": "npx cross-env NODE_OPTIONS=--experimental-vm-modules LOCAL_PATH=test/data jest --config ./test/jest-e2e.json --coverage",
+    "test:all": "npx cross-env NODE_OPTIONS=--experimental-vm-modules LIBMS_CONFIG_PATH=test/libms.test.yaml jest --testPathIgnorePatterns=cloudcmd --coverage",
+    "test:e2e": "npx cross-env NODE_OPTIONS=--experimental-vm-modules LIBMS_CONFIG_PATH=test/libms.test.yaml jest --config ./test/jest-e2e.json --coverage",
     "test:http": "yarn build && pm2 start -f --name libms-test dist/src/main.js -- -c .env -H ./config/http.json && jest test/cloudcmd --coverage --coverageThreshold=\"{}\" && pm2 delete libms-test",
     "test:http-nocov": "yarn build && pm2 start -f --name libms-test dist/src/main.js -- -c .env -H ./config/http.json && jest test/cloudcmd  --coverage=false && pm2 delete libms-test",
     "test:http-github": "jest test/cloudcmd --coverage --coverageThreshold=\"{}\"",
-    "test:int": "npx cross-env NODE_OPTIONS=--experimental-vm-modules LOCAL_PATH=test/data jest ../test/integration --coverage",
-    "test:nocov": "yarn test:http-nocov && npx cross-env LOCAL_PATH=test/data jest --testPathIgnorePatterns=cloudcmd --coverage=false",
-    "test:unit": "npx cross-env NODE_OPTIONS=--experimental-vm-modules LOCAL_PATH=test/data jest ../test/unit --coverage"
+    "test:int": "npx cross-env NODE_OPTIONS=--experimental-vm-modules LIBMS_CONFIG_PATH=test/libms.test.yaml jest ../test/integration --coverage",
+    "test:nocov": "yarn test:http-nocov && npx cross-env LIBMS_CONFIG_PATH=test/libms.test.yaml jest --testPathIgnorePatterns=cloudcmd --coverage=false",
+    "test:unit": "npx cross-env NODE_OPTIONS=--experimental-vm-modules LIBMS_CONFIG_PATH=test/libms.test.yaml jest ../test/unit --coverage"
   },
   "bin": "./dist/src/main.js",
   "dependencies": {
@@ -47,8 +47,10 @@
     "cloudcmd": "^18.1.0",
     "commander": "^12.1.0",
     "dotenv": "^16.4.5",
-    "globals": "^15.9.0",
+    "globals": "^15.11.0",
     "graphql": "^16.9.0",
+    "isomorphic-git": "^1.27.1",
+    "js-yaml": "^4.1.0",
     "mock-fs": "^5.3.0",
     "reflect-metadata": "^0.2.2",
     "rxjs": "^7.8.1",
@@ -56,7 +58,7 @@
     "type-graphql": "^2.0.0-rc.2"
   },
   "devDependencies": {
-    "@eslint/compat": "^1.1.1",
+    "@eslint/compat": "^1.2.2",
     "@eslint/eslintrc": "^3.1.0",
     "@nestjs/cli": "^10.4.5",
     "@nestjs/schematics": "^10.1.4",
@@ -65,8 +67,8 @@
     "@types/jest": "^29.5.13",
     "@types/node": "22.7.0",
     "@types/supertest": "^6.0.2",
-    "@typescript-eslint/eslint-plugin": "^8.8.0",
-    "@typescript-eslint/parser": "^8.8.0",
+    "@typescript-eslint/eslint-plugin": "^8.12.2",
+    "@typescript-eslint/parser": "^8.12.2",
     "cross-fetch": "^4.0.0",
     "eslint": "^9.11.1",
     "eslint-config-airbnb-base": "^15.0.0",
@@ -82,6 +84,7 @@
     "ts-jest": "^29.2.5",
     "ts-node": "^10.9.2",
     "tsconfig-paths": "4.2.0",
-    "typescript": "^5.6.2"
+    "typescript": "^5.6.3",
+    "@types/js-yaml": "^4.0.9"
   }
 }

package/src/app.module.ts

@@ -1,22 +1,21 @@
-import { ConfigModule, ConfigService } from '@nestjs/config';
 import { Module } from '@nestjs/common';
 import { GraphQLModule } from '@nestjs/graphql';
 import { ApolloDriver } from '@nestjs/apollo';
 import { join } from 'path';
 import FilesModule from './files/files.module.js';
+import { ConfigModule } from './config/config.module.js';
+import { CONFIG_SERVICE, IConfig } from './config/config.interface.js';
 
 @Module({
   imports: [
-    ConfigModule.forRoot({
-      isGlobal: true,
-    }),
+    ConfigModule,
     GraphQLModule.forRootAsync({
       driver: ApolloDriver,
-      useFactory: (configService: ConfigService) => ({
+      useFactory: (configService: IConfig) => ({
         autoSchemaFile: join(process.cwd(), 'src/schema.gql'),
-        path: configService.get<string>('APOLLO_PATH'),
+        path: configService.getApolloPath(),
       }),
-      inject: [ConfigService],
+      inject: [CONFIG_SERVICE],
     }),
     FilesModule,
   ],

package/src/bootstrap.ts

@@ -1,8 +1,8 @@
 import { NestFactory } from '@nestjs/core';
-import { ConfigService } from '@nestjs/config';
-import * as dotenv from 'dotenv';
 import AppModule from './app.module.js';
 import cloudCMD from './cloudcmd/cloudcmd.js';
+import { Logger } from '@nestjs/common';
+import { CONFIG_SERVICE, IConfig } from './config/config.interface.js';
 
 type BootstrapOptions = {
   config?: string;
@@ -11,26 +11,23 @@ type BootstrapOptions = {
 };
 
 export default async function bootstrap(options?: BootstrapOptions) {
-  const configFile = dotenv.config({
-    path: options?.config ?? '.env',
-    override: true,
-  });
-  if (configFile.error && process.env.LOCAL_PATH === undefined) {
-    // eslint-disable-next-line no-console
-    console.error(configFile.error);
-    if (options.runHelp) {
-      options.runHelp();
-    } else {
-      process.exit(1);
-    }
-  }
+  const logger = new Logger(bootstrap.name);
+
+  if (process.env.LIBMS_CONFIG_PATH == null)
+    process.env.LIBMS_CONFIG_PATH = options.config ?? 'libms.yaml';
 
   const app = await NestFactory.create(AppModule);
-  const configService = app.get(ConfigService);
-  const port = configService.get<number>('PORT');
+  const configService = app.get(CONFIG_SERVICE) as IConfig;
+  const port = configService.getPort();
+  const localPath = configService.getLocalPath();
+  const mode = configService.getMode();
+
+  logger.log(
+    `\x1b[32mStarting libms in \x1b[33m${mode} \x1b[32mmode, serving files from \x1b[34m${localPath} \x1b[32mon port \x1b[35m${port}\x1b[0m`,
+  );
 
   if (options.httpServer) {
-    cloudCMD(app, options.httpServer, configService.get<string>('LOCAL_PATH'));
+    cloudCMD(app, options.httpServer, configService.getLocalPath());
   }
 
   await app.listen(port);

package/src/config/config.interface.ts

@@ -0,0 +1,14 @@
+import { GitRepo } from './config.model';
+
+export const CONFIG_SERVICE = 'CONFIG_SERVICE';
+
+export interface IConfig {
+  loadConfig(configPath: string): Promise<void>;
+  getLocalPath(): string;
+  getApolloPath(): string;
+  getMode(): string;
+  getPort(): number;
+  getLogLevel(): string;
+  getGraphqlPlayground(): string;
+  getGitRepos(): { [key: string]: GitRepo }[];
+}

package/src/config/config.model.ts

@@ -0,0 +1,13 @@
+export type ConfigValues = {
+  port: number;
+  'local-path': string;
+  mode: string;
+  'log-level': string;
+  'apollo-path': string;
+  'graphql-playground': string;
+  'git-repos': { [key: string]: GitRepo }[];
+};
+
+export type GitRepo = {
+  'repo-url': string;
+};

package/src/config/config.module.ts

@@ -0,0 +1,23 @@
+import { Global, Module } from '@nestjs/common';
+import * as nestConfig from '@nestjs/config';
+import Config from './config.service.js';
+import { CONFIG_SERVICE } from './config.interface.js';
+
+@Global()
+@Module({
+  imports: [nestConfig.ConfigModule.forRoot({ isGlobal: true })],
+  providers: [
+    {
+      provide: CONFIG_SERVICE,
+      useFactory: async (envConfigservice: nestConfig.ConfigService) => {
+        const confPath = envConfigservice.get<string>('LIBMS_CONFIG_PATH');
+        const config = new Config();
+        await config.loadConfig(confPath);
+        return config;
+      },
+      inject: [nestConfig.ConfigService],
+    },
+  ],
+  exports: [CONFIG_SERVICE],
+})
+export class ConfigModule {}

package/src/config/config.service.ts

@@ -0,0 +1,54 @@
+import { readFileSync } from 'fs';
+import * as yaml from 'js-yaml';
+import { Injectable, Logger } from '@nestjs/common';
+import { ConfigValues, GitRepo } from './config.model.js';
+import resolveFile from './util.js';
+import { IConfig } from './config.interface.js';
+
+@Injectable()
+export default class Config implements IConfig {
+  private configValues: ConfigValues;
+  private logger: Logger;
+
+  constructor() {
+    this.logger = new Logger(Config.name);
+  }
+
+  async loadConfig(configPath: string): Promise<void> {
+    if (configPath !== undefined) {
+      try {
+        const configFile = readFileSync(resolveFile(configPath), 'utf8');
+        this.configValues = yaml.load(configFile) as ConfigValues;
+      } catch (e) {
+        this.logger.error('Error loading config file', e);
+        process.exit(1);
+      }
+    }
+    this.logger.log('Config loaded', this.configValues);
+  }
+
+  getLocalPath(): string {
+    return this.configValues['local-path'];
+  }
+
+  getApolloPath(): string {
+    return this.configValues['apollo-path'];
+  }
+
+  getMode(): string {
+    return this.configValues['mode'];
+  }
+
+  getPort(): number {
+    return this.configValues['port'];
+  }
+  getLogLevel(): string {
+    return this.configValues['log-level'];
+  }
+  getGraphqlPlayground(): string {
+    return this.configValues['graphql-playground'];
+  }
+  getGitRepos(): { [key: string]: GitRepo }[] {
+    return this.configValues['git-repos'];
+  }
+}

package/src/config/util.ts

@@ -0,0 +1,5 @@
+import path from 'node:path';
+
+export default function resolveFile(name: string): string {
+  return path.resolve(name);
+}

package/src/files/files-service.factory.ts

@@ -1,18 +1,20 @@
 import { Injectable } from '@nestjs/common';
-import { ConfigService } from '@nestjs/config';
 import { IFilesService } from './interfaces/files.service.interface.js';
+import { IConfig } from 'src/config/config.interface.js';
 
 @Injectable()
 export default class FilesServiceFactory {
   static create(
-    configService: ConfigService,
+    configService: IConfig,
     fileServices: IFilesService[],
   ): IFilesService {
-    const mode = configService.get<string>('MODE');
+    const mode = configService.getMode();
     const service = fileServices.find((s) => s.getMode() == mode);
 
     if (service == undefined) {
       throw new Error(`Invalid MODE: ${mode}`);
+    } else {
+      service.init();
     }
     return service;
   }

package/src/files/files.module.ts

@@ -6,7 +6,7 @@ import LocalFilesService from './local/local-files.service.js';
 import GitFilesService from './git/git-files.service.js';
 import { FILE_SERVICE } from './interfaces/files.service.interface.js';
 import FilesServiceFactory from './files-service.factory.js';
-import { ConfigService } from '@nestjs/config';
+import { CONFIG_SERVICE, IConfig } from '../config/config.interface.js';
 
 @Module({
   imports: [LocalFilesModule, GitFilesModule],
@@ -15,14 +15,14 @@ import { ConfigService } from '@nestjs/config';
     {
       provide: FILE_SERVICE,
       useFactory: (
-        configService: ConfigService,
+        configService: IConfig,
         localFilesService: LocalFilesService,
         gitFilesService: GitFilesService,
       ) => {
         const fileServices = [localFilesService, gitFilesService];
         return FilesServiceFactory.create(configService, fileServices);
       },
-      inject: [ConfigService, LocalFilesService, GitFilesService],
+      inject: [CONFIG_SERVICE, LocalFilesService, GitFilesService],
     },
   ],
 })

package/src/files/git/git-files.module.ts

@@ -1,8 +1,9 @@
 import { Module } from '@nestjs/common';
 import GitFilesService from './git-files.service.js';
+import LocalFilesService from '../local/local-files.service.js';
 
 @Module({
-  providers: [GitFilesService],
+  providers: [GitFilesService, LocalFilesService],
   exports: [GitFilesService],
 })
 export class GitFilesModule {}

package/src/files/git/git-files.service.ts

@@ -1,20 +1,65 @@
-import { Injectable } from '@nestjs/common';
+import { Inject, Injectable, Logger } from '@nestjs/common';
 import { Project } from 'src/types.js';
 import { IFilesService } from '../interfaces/files.service.interface.js';
 import { CONFIG_MODE } from '../../enums/config-mode.enum.js';
+import * as git from 'isomorphic-git';
+import * as fs from 'fs';
+import * as http from 'isomorphic-git/http/node/index.cjs';
+import LocalFilesService from '../local/local-files.service.js';
+import Config from '../../config/config.service.js';
+import { CONFIG_SERVICE } from '../../config/config.interface.js';
+import path from 'path';
 
 @Injectable()
 export default class GitFilesService implements IFilesService {
-// eslint-disable-next-line no-useless-constructor, no-empty-function
-  constructor() {}
+  private readonly dataPath: string;
+  private readonly logger: Logger;
+  @Inject(LocalFilesService) private localFilesService: LocalFilesService;
+
+  constructor(@Inject(CONFIG_SERVICE) private configService: Config) {
+    this.dataPath = this.configService.getLocalPath();
+    this.logger = new Logger(GitFilesService.name);
+  }
+
+  init(): Promise<any> {
+    return this.cloneRepositories();
+  }
+
+  private cloneRepositories(): Promise<void[]> {
+    const userRepoConfigs = this.configService.getGitRepos();
+
+    if (!userRepoConfigs || userRepoConfigs.length === 0) {
+      throw new Error('No git repos found in config');
+    }
+
+    const clonePromises = userRepoConfigs.map((repoConf) => {
+      const user = Object.keys(repoConf)[0];
+      const repoUrl = repoConf[user]['repo-url'];
+
+      return git
+        .clone({
+          fs,
+          http,
+          dir: this.dataPath + `/${user}`,
+          gitdir: path.join(this.dataPath, 'gitdir', user, '.git'),
+          url: repoUrl.includes('.git') ? repoUrl : repoUrl + '.git',
+          singleBranch: true,
+          depth: 1,
+        })
+        .then(() => this.logger.log('done cloning ' + repoUrl));
+    });
+    return Promise.all(clonePromises);
+  }
+
   getMode(): CONFIG_MODE {
     return CONFIG_MODE.GIT;
   }
 
-  listDirectory(): Promise<Project> {
-    throw new Error('Method not implemented.');
+  listDirectory(path: string): Promise<Project> {
+    return this.localFilesService.listDirectory(path);
   }
-  readFile(): Promise<Project> {
-    throw new Error('Method not implemented.');
+
+  readFile(path: string): Promise<Project> {
+    return this.localFilesService.readFile(path);
   }
 }

package/src/files/interfaces/files.service.interface.ts

@@ -7,4 +7,5 @@ export interface IFilesService {
   listDirectory(path: string): Promise<Project>;
   readFile(path: string): Promise<Project>;
   getMode(): CONFIG_MODE;
+  init(): Promise<any>;
 }

package/src/files/local/local-files.service.ts

@@ -1,19 +1,28 @@
-import { Injectable, InternalServerErrorException } from '@nestjs/common';
+import {
+  Inject,
+  Injectable,
+  InternalServerErrorException,
+} from '@nestjs/common';
 import * as fs from 'fs';
 import { join } from 'path';
-import { ConfigService } from '@nestjs/config';
 import { Project } from 'src/types.js';
 import { IFilesService } from '../interfaces/files.service.interface.js';
 import { CONFIG_MODE } from '../../enums/config-mode.enum.js';
+import Config from '../../config/config.service.js';
+import { CONFIG_SERVICE } from '../../config/config.interface.js';
 
 @Injectable()
 export default class LocalFilesService implements IFilesService {
   private readonly dataPath: string;
 
-  constructor(private configService: ConfigService) {
-    this.dataPath = this.configService.get('LOCAL_PATH');
+  constructor(@Inject(CONFIG_SERVICE) private configService: Config) {
+    this.dataPath = this.configService.getLocalPath();
+  }
 
+  init(): Promise<any> {
+    return Promise.resolve();
   }
+
   getMode(): CONFIG_MODE {
     return CONFIG_MODE.LOCAL;
   }

package/src/main.ts

@@ -2,7 +2,7 @@
 import { Command } from 'commander';
 import bootstrap from './bootstrap.js';
 
-type ProgramOptions = {
+export type ProgramOptions = {
   config?: string;
   http?: string;
 };

package/test/cloudcmd/cloudcmd.spec.ts

@@ -7,9 +7,9 @@ describe('cloudcmd test for the application', () => {
     config();
 
     await writeFile(`${process.env.LOCAL_PATH}/test.txt`, 'content12345');
-    // eslint-disable-next-line no-promise-executor-return
-    await new Promise((resolve) => setTimeout(resolve, 8000));
-  }, 10000);
+
+    await new Promise((resolve) => setTimeout(resolve, 50000)); // This is problematic.
+  }, 55000);
 
   afterAll(async () => {
     await unlink(`${process.env.LOCAL_PATH}/test.txt`);
@@ -30,7 +30,7 @@ describe('cloudcmd test for the application', () => {
     expect(fileNames).toContain('user2');
   }, 10000);
 
-  it('should return the content of a file that is uplaoded to cloudcmd ', async () => {
+  it('should return the content of a file that is uplaoded to cloudcmd', async () => {
     const response = await axios.get(
       `http://localhost:${process.env.PORT}${process.env.APOLLO_PATH}/files/api/v1/fs/test.txt`,
     );