@codifycli/plugin-core 1.0.0-beta1

package/src/plugin/plugin.ts

@@ -0,0 +1,291 @@
+import { JSONSchemaType } from 'ajv';
+import {
+  ApplyRequestData,
+  GetResourceInfoRequestData,
+  GetResourceInfoResponseData,
+  ImportRequestData,
+  ImportResponseData,
+  InitializeRequestData,
+  InitializeResponseData,
+  MatchRequestData,
+  MatchResponseData,
+  PlanRequestData,
+  PlanResponseData,
+  ResourceConfig,
+  ResourceJson, SetVerbosityRequestData,
+  ValidateRequestData,
+  ValidateResponseData
+} from 'codify-schemas';
+
+import { ApplyValidationError } from '../common/errors.js';
+import { Plan } from '../plan/plan.js';
+import { BackgroundPty } from '../pty/background-pty.js';
+import { getPty } from '../pty/index.js';
+import { SequentialPty } from '../pty/seqeuntial-pty.js';
+import { Resource } from '../resource/resource.js';
+import { ResourceController } from '../resource/resource-controller.js';
+import { ptyLocalStorage } from '../utils/pty-local-storage.js';
+import { VerbosityLevel } from '../utils/verbosity-level.js';
+
+export class Plugin {
+  planStorage: Map<string, Plan<any>>;
+  planPty = new BackgroundPty();
+
+  constructor(
+    public name: string,
+    public resourceControllers: Map<string, ResourceController<ResourceConfig>>
+  ) {
+    this.planStorage = new Map();
+  }
+
+  static create(name: string, resources: Resource<any>[]) {
+    const controllers = resources
+      .map((resource) => new ResourceController(resource))
+
+    const controllersMap = new Map<string, ResourceController<any>>(
+      controllers.map((r) => [r.typeId, r] as const)
+    );
+
+    return new Plugin(name, controllersMap);
+  }
+
+  async initialize(data: InitializeRequestData): Promise<InitializeResponseData> {
+    if (data.verbosityLevel) {
+      VerbosityLevel.set(data.verbosityLevel);
+    }
+
+    for (const controller of this.resourceControllers.values()) {
+      await controller.initialize();
+    }
+
+    return {
+      resourceDefinitions: [...this.resourceControllers.values()]
+        .map((r) => {
+          const sensitiveParameters = Object.entries(r.settings.parameterSettings ?? {})
+            .filter(([, v]) => v?.isSensitive)
+            .map(([k]) => k);
+
+          // Here we add '*' if the resource is sensitive but no sensitive parameters are found. This works because the import
+          // sensitivity check only checks for the existance of a sensitive parameter whereas the parameter blocking one blocks
+          // on a specific sensitive parameter.
+          if (r.settings.isSensitive && sensitiveParameters.length === 0) {
+            sensitiveParameters.push('*');
+          }
+
+          return {
+            dependencies: r.dependencies,
+            type: r.typeId,
+            sensitiveParameters,
+            operatingSystems: r.settings.operatingSystems,
+            linuxDistros: r.settings.linuxDistros,
+          }
+        })
+    }
+  }
+
+  getResourceInfo(data: GetResourceInfoRequestData): GetResourceInfoResponseData {
+    if (!this.resourceControllers.has(data.type)) {
+      throw new Error(`Cannot get info for resource ${data.type}, resource doesn't exist`);
+    }
+
+    const resource = this.resourceControllers.get(data.type)!;
+
+    const schema = resource.parsedSettings.schema as JSONSchemaType<any> | undefined;
+    const requiredPropertyNames = (
+      resource.settings.importAndDestroy?.requiredParameters
+      ?? (typeof resource.settings.allowMultiple === 'object' ? resource.settings.allowMultiple.identifyingParameters : null)
+      ?? schema?.required
+      ?? undefined
+    ) as any;
+
+    const allowMultiple = resource.settings.allowMultiple !== undefined
+      && resource.settings.allowMultiple !== false;
+
+    // Here we add '*' if the resource is sensitive but no sensitive parameters are found. This works because the import
+    // sensitivity check only checks for the existance of a sensitive parameter whereas the parameter blocking one blocks
+    // on a specific sensitive parameter.
+    const sensitiveParameters = Object.entries(resource.settings.parameterSettings ?? {})
+      .filter(([, v]) => v?.isSensitive)
+      .map(([k]) => k);
+
+    if (resource.settings.isSensitive && sensitiveParameters.length === 0) {
+      sensitiveParameters.push('*');
+    }
+
+    return {
+      plugin: this.name,
+      type: data.type,
+      dependencies: resource.dependencies,
+      schema: schema as Record<string, unknown> | undefined,
+      importAndDestroy: {
+        preventImport: resource.settings.importAndDestroy?.preventImport,
+        requiredParameters: requiredPropertyNames,
+      },
+      import: {
+        requiredParameters: requiredPropertyNames,
+      },
+      operatingSystems: resource.settings.operatingSystems,
+      linuxDistros: resource.settings.linuxDistros,
+      sensitiveParameters,
+      allowMultiple
+    }
+  }
+
+  async match(data: MatchRequestData): Promise<MatchResponseData> {
+    const { resource: resourceConfig, array } = data;
+
+    const resource = this.resourceControllers.get(resourceConfig.core.type);
+    if (!resource) {
+      throw new Error(`Resource of type ${resourceConfig.core.type} could not be found for match`);
+    }
+
+    const match = await resource.match(resourceConfig, array);
+    return { match }
+  }
+
+  async import(data: ImportRequestData): Promise<ImportResponseData> {
+    const { core, parameters, autoSearchAll } = data;
+
+    if (!this.resourceControllers.has(core.type)) {
+      throw new Error(`Cannot get info for resource ${core.type}, resource doesn't exist`);
+    }
+
+    const result = await ptyLocalStorage.run(this.planPty, () =>
+      this.resourceControllers
+        .get(core.type!)
+        ?.import(core, parameters, autoSearchAll)
+    )
+
+    return {
+      request: data,
+      result: result ?? [],
+    }
+  }
+
+  async validate(data: ValidateRequestData): Promise<ValidateResponseData> {
+    const validationResults: ValidateResponseData['resourceValidations'] = [];
+    for (const config of data.configs) {
+      const { core, parameters } = config;
+
+      if (!this.resourceControllers.has(core.type)) {
+        throw new Error(`Resource type not found: ${core.type}`);
+      }
+
+      const validation = await this.resourceControllers
+        .get(core.type)!
+        .validate(core, parameters);
+
+      validationResults.push(validation);
+    }
+
+    // Validate that if allow multiple is false, then only 1 of each resource exists
+    const countMap = data.configs.reduce((map, resource) => {
+      if (!map.has(resource.core.type)) {
+        map.set(resource.core.type, 0);
+      }
+
+      const count = map.get(resource.core.type)!;
+      map.set(resource.core.type, count + 1)
+
+      return map;
+    }, new Map<string, number>())
+
+    const invalidMultipleConfigs = [...countMap.entries()].filter(([k, v]) => {
+      const controller = this.resourceControllers.get(k)!;
+      return !controller.parsedSettings.allowMultiple && v > 1;
+    });
+
+    if (invalidMultipleConfigs.length > 0) {
+      throw new Error(
+        `Multiples of the following configs were found but only 1 is allowed. [${invalidMultipleConfigs.map(([k, v]) => `${v}x ${k}`).join(', ')}] found.`)
+    }
+
+    await this.crossValidateResources(data.configs);
+    return {
+      resourceValidations: validationResults
+    };
+  }
+
+  async plan(data: PlanRequestData): Promise<PlanResponseData> {
+    const { type } = data.core
+
+    if (!this.resourceControllers.has(type)) {
+      throw new Error(`Resource type not found: ${type}`);
+    }
+
+    const plan = await ptyLocalStorage.run(this.planPty, async () => this.resourceControllers.get(type)!.plan(
+      data.core,
+      data.desired ?? null,
+      data.state ?? null,
+      data.isStateful
+    ))
+
+    this.planStorage.set(plan.id, plan);
+
+    return plan.toResponse();
+  }
+
+  async apply(data: ApplyRequestData): Promise<void> {
+    if (!data.planId && !data.plan) {
+      throw new Error('For applies either plan or planId must be supplied');
+    }
+
+    const plan = this.resolvePlan(data);
+
+    const resource = this.resourceControllers.get(plan.getResourceType());
+    if (!resource) {
+      throw new Error('Malformed plan with resource that cannot be found');
+    }
+
+    await ptyLocalStorage.run(new SequentialPty(), async () => resource.apply(plan))
+
+    // Validate using desired/desired. If the apply was successful, no changes should be reported back.
+    // Default back desired back to current if it is not defined (for destroys only)
+    const validationPlan = await ptyLocalStorage.run(new BackgroundPty(), async () => {
+      const result = await resource.plan(
+        plan.coreParameters,
+        plan.desiredConfig,
+        plan.desiredConfig ?? plan.currentConfig,
+        plan.isStateful,
+        'validationPlan'
+      );
+
+      await getPty().kill();
+      return result;
+    })
+
+    if (validationPlan.requiresChanges()) {
+      throw new ApplyValidationError(plan);
+    }
+  }
+
+  async setVerbosityLevel(data: SetVerbosityRequestData): Promise<void> {
+    VerbosityLevel.set(data.verbosityLevel);
+  }
+
+  async kill() {
+    await this.planPty.kill();
+  }
+
+  private resolvePlan(data: ApplyRequestData): Plan<ResourceConfig> {
+    const { plan: planRequest, planId } = data;
+
+    if (planId) {
+      if (!this.planStorage.has(planId)) {
+        throw new Error(`Plan with id: ${planId} was not found`);
+      }
+
+      return this.planStorage.get(planId)!
+    }
+
+    if (!planRequest?.resourceType || !this.resourceControllers.has(planRequest.resourceType)) {
+      throw new Error('Malformed plan. Resource type must be supplied or resource type was not found');
+    }
+
+    const resource = this.resourceControllers.get(planRequest.resourceType)!;
+    return Plan.fromResponse(planRequest, resource.parsedSettings.defaultValues);
+  }
+
+  protected async crossValidateResources(resources: ResourceJson[]): Promise<void> {
+  }
+}

package/src/pty/background-pty.test.ts

@@ -0,0 +1,69 @@
+import { describe, expect, it } from 'vitest';
+import { BackgroundPty } from './background-pty.js';
+
+describe('BackgroundPty tests', () => {
+  it('Can launch a simple command', async () => {
+    const pty = new BackgroundPty();
+
+    const result = await pty.spawnSafe('ls');
+    expect(result).toMatchObject({
+      status: 'success',
+      exitCode: 0,
+    })
+
+
+    const exitCode = await pty.kill();
+    expect(exitCode).toMatchObject({
+      exitCode: 0,
+    });
+  })
+
+  // This test takes forever so going to disable for now.
+  // it('Can launch 100 commands in parallel', { timeout: 15000 }, async () => {
+  //   const pty = new BackgroundPty();
+  //
+  //   const fn = async () => pty.spawnSafe('ls');
+  //
+  //   const results = await Promise.all(
+  //     Array.from({ length: 100 }, (_, i) => i + 1)
+  //       .map(() => fn())
+  //   )
+  //
+  //   expect(results.length).to.eq(100);
+  //   expect(results.every((r) => r.exitCode === 0))
+  //
+  //   await pty.kill();
+  // })
+
+  it('Reports back the correct exit code and status', async () => {
+    const pty = new BackgroundPty();
+
+    const resultSuccess = await pty.spawnSafe('ls');
+    expect(resultSuccess).toMatchObject({
+      status: 'success',
+      exitCode: 0,
+    })
+
+    const resultFailed = await pty.spawnSafe('which sjkdhsakjdhjkash');
+    expect(resultFailed).toMatchObject({
+      status: 'error',
+      exitCode: 1,
+      data: 'sjkdhsakjdhjkash not found' // This might change on different os or shells. Keep for now.
+    })
+
+    await pty.kill();
+  });
+
+  it('Can use a different cwd', async () => {
+    const pty = new BackgroundPty();
+
+    const resultSuccess = await pty.spawnSafe('pwd', { cwd: '/tmp' });
+    expect(resultSuccess).toMatchObject({
+      status: 'success',
+      exitCode: 0,
+      data: '/tmp'
+    })
+
+    await pty.kill();
+  });
+})

package/src/pty/background-pty.ts

@@ -0,0 +1,154 @@
+import pty from '@homebridge/node-pty-prebuilt-multiarch';
+import { nanoid } from 'nanoid';
+import * as cp from 'node:child_process';
+import { EventEmitter } from 'node:events';
+import * as fs from 'node:fs/promises';
+import stripAnsi from 'strip-ansi';
+
+import { debugLog } from '../utils/debug.js';
+import { Shell, Utils } from '../utils/index.js';
+import { VerbosityLevel } from '../utils/verbosity-level.js';
+import { IPty, SpawnError, SpawnOptions, SpawnResult } from './index.js';
+import { PromiseQueue } from './promise-queue.js';
+
+EventEmitter.defaultMaxListeners = 1000;
+
+/**
+ * The background pty is a specialized pty designed for speed. It can launch multiple tasks
+ * in parallel by moving them to the background. It attaches unix FIFO pipes to each process
+ * to listen to stdout and stderr. One limitation of the BackgroundPty is that the tasks run
+ * without a tty (or even a stdin) attached so interactive commands will not work.
+ */
+export class BackgroundPty implements IPty {
+  private historyIgnore = Utils.getShell() === Shell.ZSH ? { HISTORY_IGNORE: '*' } : { HISTIGNORE: '*' };
+  private basePty = pty.spawn(this.getDefaultShell(), ['-i'], {
+    env: { ...process.env, ...this.historyIgnore },
+    cols: 10_000, // Set to a really large value to prevent wrapping
+    name: nanoid(6),
+    handleFlowControl: true
+  });
+
+  private promiseQueue = new PromiseQueue();
+
+  constructor() {
+    this.initialize();
+  }
+
+  async spawn(cmd: string | string[], options?: SpawnOptions): Promise<SpawnResult> {
+    const spawnResult = await this.spawnSafe(cmd, options);
+
+    if (spawnResult.status !== 'success') {
+      throw new SpawnError(Array.isArray(cmd) ? cmd.join(' ') : cmd, spawnResult.exitCode, spawnResult.data);
+    }
+
+    return spawnResult;
+  }
+
+  async spawnSafe(cmd: string | string[], options?: SpawnOptions): Promise<SpawnResult> {
+    cmd = Array.isArray(cmd) ? cmd.join('\\\n') : cmd;
+
+    // cid is command id
+    const cid = nanoid(10);
+    debugLog(cid);
+
+    await new Promise((resolve) => {
+      // 600 permissions means only the current user will be able to rw from the FIFO
+      // Create in /tmp so it could be automatically cleaned up if the clean-up was missed
+      const mkfifoSpawn = cp.spawn('mkfifo', ['-m', '600', `/tmp/${cid}`]);
+      mkfifoSpawn.on('close', () => {
+        resolve(null);
+      })
+    })
+
+    return new Promise<SpawnResult>((resolve) => {
+      const cat = cp.spawn('cat', [`/tmp/${cid}`])
+
+      let output = '';
+      cat.stdout.on('data', (data) => {
+        output += data.toString();
+
+        if (output.includes('%%%done%%%"')) {
+          const truncOutput = output.replace('%%%done%%%"\n', '');
+          const [data, exit] = truncOutput.split('%%%');
+
+          // Clean up trailing \n newline if it exists
+          let strippedData = stripAnsi(data);
+          if (strippedData.endsWith('\n')) {
+            strippedData = strippedData.slice(0, -1);
+          }
+
+          resolve(<SpawnResult>{
+            status: Number.parseInt(exit ?? 1, 10) === 0 ? 'success' : 'error',
+            exitCode: Number.parseInt(exit ?? 1, 10),
+            data: strippedData,
+          });
+        } else {
+          // Print to stdout if the verbosity level is above 0
+          if (VerbosityLevel.get() > 0) {
+            process.stdout.write(data);
+          }
+        }
+      })
+
+      this.promiseQueue.run(async () => new Promise((resolve) => {
+        const cdCommand = options?.cwd ? `cd ${options.cwd}; ` : '';
+        // Redirecting everything to the pipe and running in theb background avoids most if not all back-pressure problems
+        // Done is used to denote the end of the command
+        // Use the \\" at the end differentiate between command and response. \\" will evaluate to " in the terminal
+        const command = ` ((${cdCommand}${cmd}; echo %%%$?%%%done%%%\\") > "/tmp/${cid}" 2>&1 &); echo %%%done%%%${cid}\\";`
+
+        let output = '';
+        const listener = this.basePty.onData((data: any) => {
+          output += data;
+
+          if (output.includes(`%%%done%%%${cid}"`)) {
+            listener.dispose();
+            resolve(null);
+          }
+        });
+
+        console.log(`Running command: ${cmd}${options?.cwd ? ` (cwd: ${options.cwd})` : ''}`)
+        this.basePty.write(`${command}\r`);
+
+      }));
+    }).finally(async () => {
+      await fs.rm(`/tmp/${cid}`);
+    })
+  }
+
+  async kill(): Promise<{ exitCode: number, signal?: number | undefined }> {
+    return new Promise((resolve) => {
+      this.basePty.onExit((status) => {
+        resolve(status);
+      })
+
+      this.basePty.kill('SIGKILL')
+    })
+  }
+
+  private async initialize() {
+    // this.basePty.onData((data: string) => process.stdout.write(data));
+
+    await this.promiseQueue.run(async () => {
+      let outputBuffer = '';
+
+      return new Promise(resolve => {
+        this.basePty.write(' unset PS1;\n');
+        this.basePty.write(' unset PS0;\n')
+        this.basePty.write(' echo setup complete\\"\n')
+
+        const listener = this.basePty.onData((data: string) => {
+          outputBuffer += data;
+          if (outputBuffer.includes('setup complete"')) {
+            listener.dispose();
+            resolve(null);
+          }
+        })
+      })
+    })
+  }
+
+  private getDefaultShell(): string {
+    return process.env.SHELL!;
+  }
+}

package/src/pty/index.test.ts

@@ -0,0 +1,129 @@
+import { describe, expect, it, vitest } from 'vitest';
+import { TestConfig, TestResource } from '../utils/test-utils.test.js';
+import { getPty, IPty } from './index.js';
+import { Plugin } from '../plugin/plugin.js'
+import { CreatePlan } from '../plan/plan-types.js';
+import { OS, ResourceOperation } from 'codify-schemas';
+import { ResourceSettings } from '../resource/resource-settings.js';
+import { SequentialPty } from './seqeuntial-pty.js';
+
+describe('General tests for PTYs', () => {
+  it('Can get pty within refresh', async () => {
+    const testResource = new class extends TestResource {
+      async refresh(): Promise<Partial<TestConfig> | null> {
+        const $ = getPty();
+        const lsResult = await $.spawnSafe('ls');
+
+        expect(lsResult.exitCode).to.eq(0);
+        expect(lsResult.data).to.be.not.null;
+        expect(lsResult.status).to.eq('success');
+
+        return {};
+      }
+    }
+
+    const spy = vitest.spyOn(testResource, 'refresh')
+
+    const plugin = Plugin.create('test plugin', [testResource])
+    const plan = await plugin.plan({
+      core: { type: 'type' },
+      desired: {},
+      state: undefined,
+      isStateful: false,
+    })
+
+    expect(plan).toMatchObject({
+      operation: 'noop',
+      resourceType: 'type',
+    })
+    expect(spy).toHaveBeenCalledOnce()
+  })
+
+  it('The same pty instance is shared cross multiple resources', async () => {
+    let pty1: IPty;
+    let pty2: IPty;
+
+    const testResource1 = new class extends TestResource {
+      getSettings(): ResourceSettings<TestConfig> {
+        return {
+          id: 'type1',
+          operatingSystems: [OS.Darwin],
+        }
+      }
+
+      async refresh(): Promise<Partial<TestConfig> | null> {
+        const $ = getPty();
+        const lsResult = await $.spawnSafe('ls');
+
+        expect(lsResult.exitCode).to.eq(0);
+        pty1 = $;
+
+        return {};
+      }
+    }
+
+    const testResource2 = new class extends TestResource {
+      getSettings(): ResourceSettings<TestConfig> {
+        return {
+          id: 'type2',
+          operatingSystems: [OS.Darwin],
+        }
+      }
+
+      async refresh(): Promise<Partial<TestConfig> | null> {
+        const $ = getPty();
+        const pwdResult = await $.spawnSafe('pwd');
+
+        expect(pwdResult.exitCode).to.eq(0);
+        pty2 = $;
+
+        return {};
+      }
+    }
+
+    const spy1 = vitest.spyOn(testResource1, 'refresh')
+    const spy2 = vitest.spyOn(testResource2, 'refresh')
+
+    const plugin = Plugin.create('test plugin', [testResource1, testResource2]);
+    await plugin.plan({
+      core: { type: 'type1' },
+      desired: {},
+      state: undefined,
+      isStateful: false,
+    })
+
+    await plugin.plan({
+      core: { type: 'type2' },
+      desired: {},
+      state: undefined,
+      isStateful: false,
+    })
+
+    expect(spy1).toHaveBeenCalledOnce();
+    expect(spy2).toHaveBeenCalledOnce();
+
+    // The main check here is that the refresh method for both are sharing the same pty instance.
+    expect(pty1).to.eq(pty2);
+  })
+
+  it('Sequential pty should be available for applies', async () => {
+    const testResource = new class extends TestResource {
+      create(plan: CreatePlan<TestConfig>): Promise<void> {
+        const $ = getPty();
+        expect($).to.instanceof(SequentialPty)
+      }
+    }
+
+    const spy = vitest.spyOn(testResource, 'create')
+
+    const plugin = Plugin.create('test plugin', [testResource])
+    await plugin.apply({
+      plan: {
+        operation: ResourceOperation.CREATE,
+        resourceType: 'type',
+        parameters: [],
+      }
+    })
+    expect(spy).toHaveBeenCalledOnce()
+  })
+})