@usebruno/filestore 0.1.0

package/src/formats/bru/index.ts

@@ -0,0 +1,203 @@
+import * as _ from 'lodash';
+import {
+  bruToJsonV2,
+  jsonToBruV2,
+  bruToEnvJsonV2,
+  envJsonToBruV2,
+  collectionBruToJson as _collectionBruToJson,
+  jsonToCollectionBru as _jsonToCollectionBru
+} from '@usebruno/lang';
+
+export const bruRequestToJson = (data: string | any, parsed: boolean = false): any => {
+  try {
+    const json = parsed ? data : bruToJsonV2(data);
+
+    let requestType = _.get(json, 'meta.type');
+    if (requestType === 'http') {
+      requestType = 'http-request';
+    } else if (requestType === 'graphql') {
+      requestType = 'graphql-request';
+    } else {
+      requestType = 'http-request';
+    }
+
+    const sequence = _.get(json, 'meta.seq');
+    const transformedJson = {
+      type: requestType,
+      name: _.get(json, 'meta.name'),
+      seq: !_.isNaN(sequence) ? Number(sequence) : 1,
+      settings: _.get(json, 'settings', {}),
+      tags: _.get(json, 'meta.tags', []),
+      request: {
+        method: _.upperCase(_.get(json, 'http.method')),
+        url: _.get(json, 'http.url'),
+        params: _.get(json, 'params', []),
+        headers: _.get(json, 'headers', []),
+        auth: _.get(json, 'auth', {}),
+        body: _.get(json, 'body', {}),
+        script: _.get(json, 'script', {}),
+        vars: _.get(json, 'vars', {}),
+        assertions: _.get(json, 'assertions', []),
+        tests: _.get(json, 'tests', ''),
+        docs: _.get(json, 'docs', '')
+      }
+    };
+
+    transformedJson.request.auth.mode = _.get(json, 'http.auth', 'none');
+    transformedJson.request.body.mode = _.get(json, 'http.body', 'none');
+
+    return transformedJson;
+  } catch (e) {
+    return Promise.reject(e);
+  }
+};
+
+export const jsonRequestToBru = (json: any): string => {
+  try {
+    let type = _.get(json, 'type');
+    if (type === 'http-request') {
+      type = 'http';
+    } else if (type === 'graphql-request') {
+      type = 'graphql';
+    } else {
+      type = 'http';
+    }
+
+    const sequence = _.get(json, 'seq');
+    const bruJson = {
+      meta: {
+        name: _.get(json, 'name'),
+        type: type,
+        seq: !_.isNaN(sequence) ? Number(sequence) : 1,
+        tags: _.get(json, 'tags', []),
+      },
+      http: {
+        method: _.lowerCase(_.get(json, 'request.method')),
+        url: _.get(json, 'request.url'),
+        auth: _.get(json, 'request.auth.mode', 'none'),
+        body: _.get(json, 'request.body.mode', 'none')
+      },
+      params: _.get(json, 'request.params', []),
+      headers: _.get(json, 'request.headers', []),
+      auth: _.get(json, 'request.auth', {}),
+      body: _.get(json, 'request.body', {}),
+      script: _.get(json, 'request.script', {}),
+      vars: {
+        req: _.get(json, 'request.vars.req', []),
+        res: _.get(json, 'request.vars.res', [])
+      },
+      assertions: _.get(json, 'request.assertions', []),
+      tests: _.get(json, 'request.tests', ''),
+      settings: _.get(json, 'settings', {}),
+      docs: _.get(json, 'request.docs', '')
+    };
+
+    const bru = jsonToBruV2(bruJson);
+    return bru;
+  } catch (error) {
+    throw error;
+  }
+};
+
+export const bruCollectionToJson = (data: string | any, parsed: boolean = false): any => {
+  try {
+    const json = parsed ? data : _collectionBruToJson(data);
+
+    const transformedJson: any = {
+      request: {
+        headers: _.get(json, 'headers', []),
+        auth: _.get(json, 'auth', {}),
+        script: _.get(json, 'script', {}),
+        vars: _.get(json, 'vars', {}),
+        tests: _.get(json, 'tests', '')
+      },
+      settings: _.get(json, 'settings', {}),
+      docs: _.get(json, 'docs', '')
+    };
+
+    // add meta if it exists
+    // this is only for folder bru file
+    if (json.meta) {
+      transformedJson.meta = {
+        name: json.meta.name
+      };
+      
+      // Include seq if it exists
+      if (json.meta.seq !== undefined) {
+        const sequence = json.meta.seq;
+        transformedJson.meta.seq = !isNaN(sequence) ? Number(sequence) : 1;
+      }
+    }
+
+    return transformedJson;
+  } catch (error) {
+    return Promise.reject(error);
+  }
+};
+
+export const jsonCollectionToBru = (json: any, isFolder?: boolean): string => {
+  try {
+    const collectionBruJson: any = {
+      headers: _.get(json, 'request.headers', []),
+      script: {
+        req: _.get(json, 'request.script.req', ''),
+        res: _.get(json, 'request.script.res', '')
+      },
+      vars: {
+        req: _.get(json, 'request.vars.req', []),
+        res: _.get(json, 'request.vars.res', [])
+      },
+      tests: _.get(json, 'request.tests', ''),
+      auth: _.get(json, 'request.auth', {}),
+      docs: _.get(json, 'docs', '')
+    };
+
+    // add meta if it exists
+    // this is only for folder bru file
+    if (json?.meta) {
+      collectionBruJson.meta = {
+        name: json.meta.name
+      };
+      
+      // Include seq if it exists
+      if (json.meta.seq !== undefined) {
+        const sequence = json.meta.seq;
+        collectionBruJson.meta.seq = !isNaN(sequence) ? Number(sequence) : 1;
+      }
+    }
+
+    if (!isFolder) {
+      collectionBruJson.auth = _.get(json, 'request.auth', {});
+    }
+
+    return _jsonToCollectionBru(collectionBruJson);
+  } catch (error) {
+    throw error;
+  }
+};
+
+export const bruEnvironmentToJson = (bru: string): any => {
+  try {
+    const json = bruToEnvJsonV2(bru);
+
+    // the app env format requires each variable to have a type
+    // this need to be evaluated and safely removed
+    // i don't see it being used in schema validation
+    if (json && json.variables && json.variables.length) {
+      _.each(json.variables, (v: any) => (v.type = 'text'));
+    }
+
+    return json;
+  } catch (error) {
+    return Promise.reject(error);
+  }
+};
+
+export const jsonEnvironmentToBru = (json: any): string => {
+  try {
+    const bru = envJsonToBruV2(json);
+    return bru;
+  } catch (error) {
+    throw error;
+  }
+}; 

package/src/index.ts

@@ -0,0 +1,100 @@
+import {
+  bruRequestToJson,
+  jsonRequestToBru,
+  bruCollectionToJson,
+  jsonCollectionToBru,
+  bruEnvironmentToJson,
+  jsonEnvironmentToBru
+} from './formats/bru';
+import { dotenvToJson } from '@usebruno/lang';
+import BruParserWorker from './workers';
+import {
+  ParseOptions,
+  StringifyOptions,
+  ParsedRequest,
+  ParsedCollection,
+  ParsedEnvironment
+} from './types';
+
+export const parseRequest = (content: string, options: ParseOptions = { format: 'bru' }): any => {
+  if (options.format === 'bru') {
+    return bruRequestToJson(content);
+  }
+  throw new Error(`Unsupported format: ${options.format}`);
+};
+
+export const stringifyRequest = (requestObj: ParsedRequest, options: StringifyOptions = { format: 'bru' }): string => {
+  if (options.format === 'bru') {
+    return jsonRequestToBru(requestObj);
+  }
+  throw new Error(`Unsupported format: ${options.format}`);
+};
+
+let globalWorkerInstance: BruParserWorker | null = null;
+
+const getWorkerInstance = (): BruParserWorker => {
+  if (!globalWorkerInstance) {
+    globalWorkerInstance = new BruParserWorker();
+  }
+  return globalWorkerInstance;
+};
+
+export const parseRequestViaWorker = async (content: string): Promise<any> => {
+  const fileParserWorker = getWorkerInstance();
+  return await fileParserWorker.parseRequest(content);
+};
+
+export const stringifyRequestViaWorker = async (requestObj: any): Promise<string> => {
+  const fileParserWorker = getWorkerInstance();
+  return await fileParserWorker.stringifyRequest(requestObj);
+};
+
+export const parseCollection = (content: string, options: ParseOptions = { format: 'bru' }): any => {
+  if (options.format === 'bru') {
+    return bruCollectionToJson(content);
+  }
+  throw new Error(`Unsupported format: ${options.format}`);
+};
+
+export const stringifyCollection = (collectionObj: ParsedCollection, options: StringifyOptions = { format: 'bru' }): string => {
+  if (options.format === 'bru') {
+    return jsonCollectionToBru(collectionObj, false);
+  }
+  throw new Error(`Unsupported format: ${options.format}`);
+};
+
+export const parseFolder = (content: string, options: ParseOptions = { format: 'bru' }): any => {
+  if (options.format === 'bru') {
+    return bruCollectionToJson(content);
+  }
+  throw new Error(`Unsupported format: ${options.format}`);
+};
+
+export const stringifyFolder = (folderObj: any, options: StringifyOptions = { format: 'bru' }): string => {
+  if (options.format === 'bru') {
+    return jsonCollectionToBru(folderObj, true);
+  }
+  throw new Error(`Unsupported format: ${options.format}`);
+};
+
+export const parseEnvironment = (content: string, options: ParseOptions = { format: 'bru' }): any => {
+  if (options.format === 'bru') {
+    return bruEnvironmentToJson(content);
+  }
+  throw new Error(`Unsupported format: ${options.format}`);
+};
+
+export const stringifyEnvironment = (envObj: ParsedEnvironment, options: StringifyOptions = { format: 'bru' }): string => {
+  if (options.format === 'bru') {
+    return jsonEnvironmentToBru(envObj);
+  }
+  throw new Error(`Unsupported format: ${options.format}`);
+};
+
+
+export const parseDotEnv = (content: string): Record<string, string> => {
+  return dotenvToJson(content);
+};
+
+export { BruParserWorker };
+export * from './types';

package/src/types/bruno-lang.d.ts

@@ -0,0 +1,9 @@
+declare module '@usebruno/lang' {
+  export function bruToJsonV2(bruContent: string): any;
+  export function jsonToBruV2(jsonData: any): string;
+  export function bruToEnvJsonV2(bruContent: string): any;
+  export function envJsonToBruV2(jsonData: any): string;
+  export function collectionBruToJson(bruContent: string): any;
+  export function jsonToCollectionBru(jsonData: any): string;
+  export function dotenvToJson(envContent: string): Record<string, string>;
+} 

package/src/types.ts

@@ -0,0 +1,141 @@
+export interface ParseOptions {
+  format?: 'bru' | 'yaml';
+}
+
+export interface StringifyOptions {
+  format?: 'bru' | 'yaml';
+}
+
+export interface RequestBody {
+  mode?: string;
+  raw?: string;
+  formUrlEncoded?: Array<{ name: string; value: string; enabled: boolean }>;
+  multipartForm?: Array<{ name: string; value: string; type: string; enabled: boolean }>;
+  json?: string;
+  xml?: string;
+  sparql?: string;
+  graphql?: {
+    query?: string;
+    variables?: string;
+  };
+}
+
+export interface AuthConfig {
+  mode?: string;
+  basic?: {
+    username?: string;
+    password?: string;
+  };
+  bearer?: {
+    token?: string;
+  };
+  apikey?: {
+    key?: string;
+    value?: string;
+    placement?: string;
+  };
+  awsv4?: {
+    accessKeyId?: string;
+    secretAccessKey?: string;
+    sessionToken?: string;
+    service?: string;
+    region?: string;
+    profileName?: string;
+  };
+  oauth2?: {
+    grantType?: string;
+    callbackUrl?: string;
+    authorizationUrl?: string;
+    accessTokenUrl?: string;
+    clientId?: string;
+    clientSecret?: string;
+    scope?: string;
+    state?: string;
+    pkce?: boolean;
+  };
+}
+
+export interface RequestParam {
+  name: string;
+  value: string;
+  enabled: boolean;
+}
+
+export interface RequestHeader {
+  name: string;
+  value: string;
+  enabled: boolean;
+}
+
+export interface RequestAssertion {
+  name: string;
+  value: string;
+  enabled: boolean;
+}
+
+export interface RequestVars {
+  req?: Array<{ name: string; value: string; enabled: boolean }>;
+  res?: Array<{ name: string; value: string; enabled: boolean }>;
+}
+
+export interface RequestScript {
+  req?: string;
+  res?: string;
+}
+
+export interface RequestSettings {
+  [key: string]: any;
+}
+
+export interface RequestData {
+  method: string;
+  url: string;
+  params: RequestParam[];
+  headers: RequestHeader[];
+  auth: AuthConfig;
+  body: RequestBody;
+  script: RequestScript;
+  vars: RequestVars;
+  assertions: RequestAssertion[];
+  tests: string;
+  docs: string;
+}
+
+export interface ParsedRequest {
+  type: 'http-request' | 'graphql-request';
+  name: string;
+  seq: number;
+  settings: RequestSettings;
+  tags: string[];
+  request: RequestData;
+}
+
+export interface ParsedCollection {
+  name: string;
+  type?: string;
+  version?: string;
+  [key: string]: any;
+}
+
+export interface EnvironmentVariable {
+  name: string;
+  value: string;
+  enabled: boolean;
+}
+
+export interface ParsedEnvironment {
+  variables: EnvironmentVariable[];
+}
+
+export interface WorkerTask {
+  data: any;
+  priority: number;
+  scriptPath: string;
+  taskType?: 'parse' | 'stringify';
+  resolve?: (value: any) => void;
+  reject?: (reason?: any) => void;
+}
+
+export interface Lane {
+  maxSize: number;
+} 

package/src/workers/WorkerQueue/index.ts

@@ -0,0 +1,114 @@
+import { Worker } from 'node:worker_threads';
+
+interface QueuedTask {
+  priority: number;
+  scriptPath: string;
+  data: any;
+  taskType: 'parse' | 'stringify';
+  resolve?: (value: any) => void;
+  reject?: (reason?: any) => void;
+}
+
+class WorkerQueue {
+  private queue: QueuedTask[];
+  private isProcessing: boolean;
+  private workers: Record<string, Worker>;
+
+  constructor() {
+    this.queue = [];
+    this.isProcessing = false;
+    this.workers = {};
+  }
+
+  async getWorkerForScriptPath(scriptPath: string) {
+    if (!this.workers) this.workers = {}; 
+    let worker = this.workers[scriptPath];
+    if (!worker || worker.threadId === -1) {
+      this.workers[scriptPath] = worker = new Worker(scriptPath);
+    }
+    return worker;
+  }
+  
+  async enqueue(task: QueuedTask) {
+    const { priority, scriptPath, data, taskType } = task;
+
+    return new Promise((resolve, reject) => {
+      this.queue.push({ priority, scriptPath, data, taskType, resolve, reject });
+      this.queue?.sort((taskX, taskY) => taskX?.priority - taskY?.priority);
+      this.processQueue();
+    });
+  }
+
+  async processQueue() {
+    if (this.isProcessing || this.queue.length === 0){
+      return;
+    } 
+
+    this.isProcessing = true;
+    const { scriptPath, data, taskType, resolve, reject } = this.queue.shift() as QueuedTask;
+
+    try {
+      const result = await this.runWorker({ scriptPath, data, taskType });
+      resolve?.(result);
+    } catch (error) {
+      reject?.(error);
+    } finally {
+      this.isProcessing = false;
+      this.processQueue();
+    }
+  }
+
+  async runWorker({ scriptPath, data, taskType }: { scriptPath: string; data: any; taskType: 'parse' | 'stringify' }) {
+    return new Promise(async (resolve, reject) => {
+      let worker = await this.getWorkerForScriptPath(scriptPath);
+      
+      const messageHandler = (data: any) => {
+        worker.off('message', messageHandler);
+        worker.off('error', errorHandler);
+        worker.off('exit', exitHandler);
+        
+        if (data?.error) {
+          reject(new Error(data?.error));
+        } else {
+          resolve(data);
+        }
+      };
+
+      const errorHandler = (error: Error) => {
+        worker.off('message', messageHandler);
+        worker.off('error', errorHandler);
+        worker.off('exit', exitHandler);
+        reject(error);
+      };
+
+      const exitHandler = (code: number) => {
+        worker.off('message', messageHandler);
+        worker.off('error', errorHandler);
+        worker.off('exit', exitHandler);
+        // Remove dead worker from cache
+        delete this.workers[scriptPath];
+        reject(new Error(`Worker stopped with exit code ${code}`));
+      };
+      
+      worker.on('message', messageHandler);
+      worker.on('error', errorHandler);
+      worker.on('exit', exitHandler);
+
+      worker.postMessage({ taskType, data });
+    });
+  }
+
+  async cleanup() {
+    const promises = Object.values(this.workers).map(worker => {
+      if (worker.threadId !== -1) {
+        return worker.terminate();
+      }
+      return Promise.resolve();
+    });
+    
+    await Promise.allSettled(promises);
+    this.workers = {};
+  }
+}
+
+export default WorkerQueue;

package/src/workers/index.ts

@@ -0,0 +1,86 @@
+import WorkerQueue from './WorkerQueue';
+import { Lane } from '../types';
+import path from 'node:path';
+
+const sizeInMB = (size: number): number => {
+  return size / (1024 * 1024);
+}
+
+const getSize = (data: any): number => {
+  return sizeInMB(typeof data === 'string' ? Buffer.byteLength(data, 'utf8') : Buffer.byteLength(JSON.stringify(data), 'utf8'));
+}
+
+/**
+ * Lanes are used to determine which worker queue to use based on the size of the data.
+ * 
+ * The first lane is for smaller files (<0.1MB), the second lane is for larger files (>=0.1MB).
+ * This helps with parsing performance.
+ */
+const LANES: Lane[] = [{
+  maxSize: 0.005
+},{
+  maxSize: 0.1
+},{
+  maxSize: 1
+},{
+  maxSize: 10
+},{
+  maxSize: 100
+}];
+
+interface WorkerQueueWithSize {
+  maxSize: number;
+  workerQueue: WorkerQueue;
+
+}
+
+class BruParserWorker {
+  private workerQueues: WorkerQueueWithSize[];
+
+  constructor() {
+    this.workerQueues = LANES?.map(lane => ({
+      maxSize: lane?.maxSize,
+      workerQueue: new WorkerQueue()
+    }));
+  }
+
+  private getWorkerQueue(size: number): WorkerQueue {
+    // Find the first queue that can handle the given size
+    // or fallback to the last queue for largest files
+    const queueForSize = this.workerQueues.find((queue) => 
+      queue.maxSize >= size
+    );
+
+    return queueForSize?.workerQueue ?? this.workerQueues[this.workerQueues.length - 1].workerQueue;
+  }
+
+  private async enqueueTask({ data, taskType }: { data: any; taskType: 'parse' | 'stringify' }): Promise<any> {
+    const size = getSize(data);
+    const workerQueue = this.getWorkerQueue(size);
+    const workerScriptPath = path.join(__dirname, './workers/worker-script.js');
+    
+    return workerQueue.enqueue({
+      data,
+      priority: size,
+      scriptPath: workerScriptPath,
+      taskType,
+    });
+  }
+
+  async parseRequest(data: any): Promise<any> {
+    return this.enqueueTask({ data, taskType: 'parse' });
+  }
+
+  async stringifyRequest(data: any): Promise<any> {
+    return this.enqueueTask({ data, taskType: 'stringify' });
+  }
+
+  async cleanup(): Promise<void> {
+    const cleanupPromises = this.workerQueues.map(({ workerQueue }) => 
+      workerQueue.cleanup()
+    );
+    await Promise.allSettled(cleanupPromises);
+  }
+}
+
+export default BruParserWorker; 

package/src/workers/worker-script.ts

@@ -0,0 +1,27 @@
+import { parentPort } from 'node:worker_threads';
+import { bruRequestToJson, jsonRequestToBru } from '../formats/bru';
+
+interface WorkerMessage {
+  taskType: 'parse' | 'stringify';
+  data: any;
+}
+
+parentPort?.on('message', async (message: WorkerMessage) => {
+  try {
+    const { taskType, data } = message;
+    let result: any;
+
+    if (taskType === 'parse') {
+      result = bruRequestToJson(data);
+    } else if (taskType === 'stringify') {
+      result = jsonRequestToBru(data);
+    } else {
+      throw new Error(`Unknown task type: ${taskType}`);
+    }
+
+    parentPort?.postMessage(result);
+  } catch (error: any) {
+    console.error('Worker error:', error);
+    parentPort?.postMessage({ error: error?.message });
+  }
+});