@nospt/plugin-dev-ai-hub-backend 0.2.4 → 0.3.0

package/src/service/AiAssetSyncService.ts

@@ -1,298 +0,0 @@
-import type {
-  LoggerService,
-  SchedulerService,
-  UrlReaderService,
-} from '@backstage/backend-plugin-api';
-import type { AiAssetStore } from '../database/AiAssetStore';
-import type { ProviderConfig } from '../types';
-import type { AiAssetProvider } from '@nospt/plugin-dev-ai-hub-node';
-import { AssetParser } from './AssetParser';
-
-interface Options {
-  logger: LoggerService;
-  store: AiAssetStore;
-  scheduler: SchedulerService;
-  urlReader: UrlReaderService;
-  providers: ProviderConfig[];
-  externalProviders: AiAssetProvider[];
-}
-
-export class AiAssetSyncService {
-  constructor(private readonly options: Options) {}
-
-  async start(): Promise<void> {
-    const { scheduler, logger, providers, externalProviders } = this.options;
-
-    if (providers.length === 0 && externalProviders.length === 0) {
-      logger.warn(
-        'dev-ai-hub: no providers configured under devAiHub.providers',
-      );
-      return;
-    }
-
-    for (const provider of providers) {
-      const { frequency, timeout } = provider.schedule;
-      const frequencyDuration = frequency.minutes
-        ? { minutes: frequency.minutes }
-        : { hours: frequency.hours ?? 1 };
-      const timeoutDuration = timeout.minutes
-        ? { minutes: timeout.minutes }
-        : { hours: timeout.hours ?? 1 };
-
-      await scheduler.scheduleTask({
-        id: `dev-ai-hub-sync-${provider.id}`,
-        frequency: frequencyDuration,
-        timeout: timeoutDuration,
-        initialDelay: { seconds: 15 },
-        fn: async () => {
-          await this.syncProvider(provider);
-        },
-      });
-
-      logger.info(
-        `dev-ai-hub: scheduled sync for provider "${provider.id}" every ${JSON.stringify(frequencyDuration)}`,
-      );
-    }
-
-    for (const extProvider of externalProviders) {
-      await scheduler.scheduleTask({
-        id: `dev-ai-hub-sync-external-${extProvider.id}`,
-        frequency: { minutes: 60 },
-        timeout: { minutes: 5 },
-        initialDelay: { seconds: 15 },
-        fn: async () => {
-          await this.syncExternalProvider(extProvider);
-        },
-      });
-
-      logger.info(
-        `dev-ai-hub: scheduled sync for external provider "${extProvider.id}"`,
-      );
-    }
-  }
-
-  async syncProvider(provider: ProviderConfig): Promise<void> {
-    const { logger, urlReader, store } = this.options;
-
-    logger.info(`dev-ai-hub: starting sync for provider "${provider.id}"`);
-
-    await store.upsertSyncStatus({
-      providerId: provider.id,
-      status: 'syncing',
-      assetCount: 0,
-    });
-
-    try {
-      const treeUrl = buildTreeUrl(provider);
-      const tree = await urlReader.readTree(treeUrl);
-      const files = await tree.files();
-
-      const fileMap = new Map<
-        string,
-        { content(): Promise<Buffer>; path: string }
-      >();
-      for (const file of files) {
-        fileMap.set(normalizePath(file.path), file);
-      }
-
-      const syncedIds: string[] = [];
-
-      for (const [filePath, file] of fileMap) {
-        if (!filePath.endsWith('.yaml')) continue;
-
-        const yamlContent = (await file.content()).toString('utf-8');
-        const parsed = AssetParser.parseYaml(yamlContent, filePath);
-
-        if (!parsed) {
-          logger.warn(
-            `dev-ai-hub: skipping invalid YAML at ${filePath} (provider ${provider.id})`,
-          );
-          continue;
-        }
-
-        if (
-          provider.filters?.types &&
-          !provider.filters.types.includes(parsed.meta.type)
-        ) {
-          continue;
-        }
-        if (provider.filters?.tools) {
-          const hasMatchingTool = parsed.meta.tools.some(t =>
-            provider.filters!.tools!.includes(t),
-          );
-          if (!hasMatchingTool) continue;
-        }
-
-        // Resolve the .md file:
-        //   - explicit: yaml has a content: field → use that exact path
-        //   - junction: find the single .md co-located in the same directory
-        const yamlDir = filePath.includes('/')
-          ? filePath.split('/').slice(0, -1).join('/')
-          : '';
-
-        let resolvedMdPath: string;
-        let mdFile: { content(): Promise<Buffer>; path: string } | undefined;
-
-        if (parsed.mdPath) {
-          resolvedMdPath = normalizePath(parsed.mdPath);
-          mdFile = fileMap.get(resolvedMdPath);
-          if (!mdFile) {
-            logger.warn(
-              `dev-ai-hub: content file "${parsed.mdPath}" not found for ${filePath}, skipping`,
-            );
-            continue;
-          }
-        } else {
-          const mdCandidates = [...fileMap.keys()].filter(p => {
-            const dir = p.includes('/') ? p.split('/').slice(0, -1).join('/') : '';
-            return dir === yamlDir && p.endsWith('.md');
-          });
-          if (mdCandidates.length === 1) {
-            resolvedMdPath = mdCandidates[0];
-            mdFile = fileMap.get(resolvedMdPath)!;
-          } else if (mdCandidates.length > 1) {
-            logger.warn(
-              `dev-ai-hub: multiple .md files in "${yamlDir || '.'}" for ${filePath} — add a 'content' field to the YAML to specify which one, skipping`,
-            );
-            continue;
-          } else {
-            logger.warn(
-              `dev-ai-hub: no .md file found for ${filePath}, skipping`,
-            );
-            continue;
-          }
-        }
-
-        const mdContent = (await mdFile.content()).toString('utf-8');
-
-        // For skills: read bundled resource files and store their content
-        let resourcesContent: Record<string, string> | undefined;
-        const resourcePaths = parsed.meta.resources;
-        if (resourcePaths && resourcePaths.length > 0) {
-          resourcesContent = {};
-          for (const resourcePath of resourcePaths) {
-            const fullPath = normalizePath(
-              yamlDir ? `${yamlDir}/${resourcePath}` : resourcePath,
-            );
-            const resourceFile = fileMap.get(fullPath);
-            if (resourceFile) {
-              resourcesContent[resourcePath] = (await resourceFile.content()).toString('utf-8');
-            } else {
-              logger.warn(
-                `dev-ai-hub: resource file not found: ${fullPath} (asset ${filePath})`,
-              );
-            }
-          }
-          if (Object.keys(resourcesContent).length === 0) resourcesContent = undefined;
-        }
-
-        const asset = AssetParser.buildAsset(
-          parsed,
-          mdContent,
-          provider.id,
-          provider.target,
-          provider.branch,
-          filePath,
-          resolvedMdPath,
-          resourcesContent,
-        );
-
-        await store.upsertAsset(asset);
-        syncedIds.push(asset.id);
-      }
-
-      await store.deleteAssetsNotIn(provider.id, syncedIds);
-
-      await store.upsertSyncStatus({
-        providerId: provider.id,
-        lastSync: new Date().toISOString(),
-        status: 'idle',
-        assetCount: syncedIds.length,
-      });
-
-      logger.info(
-        `dev-ai-hub: sync complete for provider "${provider.id}" — ${syncedIds.length} assets`,
-      );
-    } catch (err) {
-      const message = err instanceof Error ? err.message : String(err);
-      this.options.logger.error(
-        `dev-ai-hub: sync failed for provider "${provider.id}": ${message}`,
-      );
-      await store.upsertSyncStatus({
-        providerId: provider.id,
-        status: 'error',
-        error: message,
-        assetCount: 0,
-      });
-    }
-  }
-
-  private async syncExternalProvider(provider: AiAssetProvider): Promise<void> {
-    const { logger, store } = this.options;
-
-    logger.info(`dev-ai-hub: syncing external provider "${provider.id}"`);
-
-    await store.upsertSyncStatus({
-      providerId: provider.id,
-      status: 'syncing',
-      assetCount: 0,
-    });
-
-    try {
-      const assets = await provider.getAssets();
-      const syncedIds: string[] = [];
-
-      for (const assetData of assets) {
-        const id = AssetParser.buildId(provider.id, assetData.yamlPath);
-        await store.upsertAsset({ ...assetData, id });
-        syncedIds.push(id);
-      }
-
-      await store.deleteAssetsNotIn(provider.id, syncedIds);
-
-      await store.upsertSyncStatus({
-        providerId: provider.id,
-        lastSync: new Date().toISOString(),
-        status: 'idle',
-        assetCount: syncedIds.length,
-      });
-
-      logger.info(
-        `dev-ai-hub: external sync complete for "${provider.id}" — ${syncedIds.length} assets`,
-      );
-    } catch (err) {
-      const message = err instanceof Error ? err.message : String(err);
-      logger.error(
-        `dev-ai-hub: external sync failed for "${provider.id}": ${message}`,
-      );
-      await store.upsertSyncStatus({
-        providerId: provider.id,
-        status: 'error',
-        error: message,
-        assetCount: 0,
-      });
-    }
-  }
-}
-
-function normalizePath(p: string): string {
-  return p.replace(/\\/g, '/').replace(/^\//, '');
-}
-
-function buildTreeUrl(provider: ProviderConfig): string {
-  const base = provider.target.replace(/\.git$/, '');
-  const { branch, type } = provider;
-
-  switch (type) {
-    case 'gitlab':
-      return `${base}/-/tree/${branch}/`;
-    case 'bitbucket':
-      return `${base}/src/${branch}/`;
-    case 'azure-devops':
-      return `${base}?version=GB${branch}`;
-    case 'github':
-    case 'git':
-    default:
-      return `${base}/tree/${branch}/`;
-  }
-}

package/src/service/AssetParser.test.ts

@@ -1,329 +0,0 @@
-import { AssetParser } from './AssetParser';
-
-const VALID_YAML = `
-name: My Instruction
-description: A test instruction
-type: instruction
-tools:
-  - claude-code
-tags:
-  - testing
-author: Test Author
-version: 1.2.3
-`;
-
-describe('AssetParser.parseYaml', () => {
-  it('parses a valid YAML envelope and returns metadata + mdPath', () => {
-    const result = AssetParser.parseYaml(VALID_YAML, 'instructions/my-instruction.yaml');
-    expect(result).not.toBeNull();
-    expect(result!.meta.name).toBe('My Instruction');
-    expect(result!.meta.description).toBe('A test instruction');
-    expect(result!.meta.type).toBe('instruction');
-    expect(result!.meta.tools).toEqual(['claude-code']);
-    expect(result!.meta.tags).toEqual(['testing']);
-    expect(result!.meta.author).toBe('Test Author');
-    expect(result!.meta.version).toBe('1.2.3');
-    expect(result!.yamlRaw).toBe(VALID_YAML);
-  });
-
-  it('leaves mdPath undefined when no content field is set', () => {
-    const result = AssetParser.parseYaml(VALID_YAML, 'instructions/my-instruction.yaml');
-    expect(result!.mdPath).toBeUndefined();
-  });
-
-  it('uses the content field to resolve mdPath when present', () => {
-    const yaml = `${VALID_YAML}content: custom-content.md\n`;
-    const result = AssetParser.parseYaml(yaml, 'instructions/my-instruction.yaml');
-    expect(result!.mdPath).toBe('instructions/custom-content.md');
-  });
-
-  it('resolves content path relative to the yaml directory', () => {
-    const yaml = `${VALID_YAML}content: sub/custom.md\n`;
-    const result = AssetParser.parseYaml(yaml, 'agents/my-agent.yaml');
-    expect(result!.mdPath).toBe('agents/sub/custom.md');
-  });
-
-  it('returns null for invalid YAML syntax', () => {
-    expect(AssetParser.parseYaml(': bad: yaml: :', 'test.yaml')).toBeNull();
-  });
-
-  it('returns null when required field "name" is missing', () => {
-    const yaml = `
-description: A description
-type: instruction
-tools:
-  - claude-code
-`;
-    expect(AssetParser.parseYaml(yaml, 'test.yaml')).toBeNull();
-  });
-
-  it('returns null when required field "type" is missing', () => {
-    const yaml = `
-name: Test
-description: A description
-tools:
-  - claude-code
-`;
-    expect(AssetParser.parseYaml(yaml, 'test.yaml')).toBeNull();
-  });
-
-  it('returns null when type is not a recognised enum value', () => {
-    const yaml = `
-name: Test
-description: A description
-type: invalid-type
-tools:
-  - claude-code
-`;
-    expect(AssetParser.parseYaml(yaml, 'test.yaml')).toBeNull();
-  });
-
-  it('returns null when tools array is empty', () => {
-    const yaml = `
-name: Test
-description: A description
-type: instruction
-tools: []
-`;
-    expect(AssetParser.parseYaml(yaml, 'test.yaml')).toBeNull();
-  });
-
-  it('applies schema defaults for optional fields', () => {
-    const yaml = `
-name: Minimal
-description: Minimal description
-type: agent
-tools:
-  - claude-code
-`;
-    const result = AssetParser.parseYaml(yaml, 'agents/minimal.yaml');
-    expect(result!.meta.author).toBe('Unknown');
-    expect(result!.meta.version).toBe('1.0.0');
-    expect(result!.meta.tags).toEqual([]);
-  });
-
-  it('passes through unrecognised fields (passthrough schema)', () => {
-    const yaml = `${VALID_YAML}unknownField: some-value\n`;
-    const result = AssetParser.parseYaml(yaml, 'test.yaml');
-    expect(result).not.toBeNull();
-  });
-});
-
-describe('AssetParser.buildId', () => {
-  it('returns a base64url encoded string', () => {
-    const id = AssetParser.buildId('provider-1', 'instructions/my-asset.yaml');
-    expect(id).toBe(Buffer.from('provider-1:instructions/my-asset.yaml').toString('base64url'));
-  });
-
-  it('normalises backslashes to forward slashes', () => {
-    const id1 = AssetParser.buildId('p', 'instructions\\my-asset.yaml');
-    const id2 = AssetParser.buildId('p', 'instructions/my-asset.yaml');
-    expect(id1).toBe(id2);
-  });
-
-  it('strips a leading slash', () => {
-    const id1 = AssetParser.buildId('p', '/instructions/my-asset.yaml');
-    const id2 = AssetParser.buildId('p', 'instructions/my-asset.yaml');
-    expect(id1).toBe(id2);
-  });
-
-  it('is deterministic for the same inputs', () => {
-    const id1 = AssetParser.buildId('provider-1', 'path/to/asset.yaml');
-    const id2 = AssetParser.buildId('provider-1', 'path/to/asset.yaml');
-    expect(id1).toBe(id2);
-  });
-
-  it('produces different IDs for different providers', () => {
-    const id1 = AssetParser.buildId('provider-1', 'path/asset.yaml');
-    const id2 = AssetParser.buildId('provider-2', 'path/asset.yaml');
-    expect(id1).not.toBe(id2);
-  });
-
-  it('produces different IDs for different paths', () => {
-    const id1 = AssetParser.buildId('p', 'instructions/a.yaml');
-    const id2 = AssetParser.buildId('p', 'agents/a.yaml');
-    expect(id1).not.toBe(id2);
-  });
-});
-
-describe('AssetParser.buildAsset', () => {
-  const parsed = AssetParser.parseYaml(VALID_YAML, 'instructions/my-instruction.yaml')!;
-
-  it('builds a complete AiAssetInput with all expected fields', () => {
-    const asset = AssetParser.buildAsset(
-      parsed,
-      '# Content',
-      'my-provider',
-      'https://github.com/org/repo',
-      'main',
-      'instructions/my-instruction.yaml',
-      'instructions/my-instruction.md',
-    );
-
-    expect(asset.id).toBe(AssetParser.buildId('my-provider', 'instructions/my-instruction.yaml'));
-    expect(asset.name).toBe('My Instruction');
-    expect(asset.type).toBe('instruction');
-    expect(asset.tools).toEqual(['claude-code']);
-    expect(asset.tags).toEqual(['testing']);
-    expect(asset.content).toBe('# Content');
-    expect(asset.yamlRaw).toBe(VALID_YAML);
-    expect(asset.providerId).toBe('my-provider');
-    expect(asset.repoUrl).toBe('https://github.com/org/repo');
-    expect(asset.branch).toBe('main');
-    expect(asset.yamlPath).toBe('instructions/my-instruction.yaml');
-    expect(asset.mdPath).toBe('instructions/my-instruction.md');
-    expect(asset.author).toBe('Test Author');
-    expect(asset.version).toBe('1.2.3');
-  });
-
-  it('sets metadata.mcpServers when mcpServers is present in the YAML', () => {
-    const yaml = `${VALID_YAML}mcpServers:\n  my-server:\n    url: http://localhost\n`;
-    const parsedWithMcp = AssetParser.parseYaml(yaml, 'agents/agent.yaml')!;
-    const asset = AssetParser.buildAsset(parsedWithMcp, '', 'p', 'url', 'main', 'agents/agent.yaml', 'agents/agent.md');
-    expect(asset.metadata?.mcpServers).toEqual({ 'my-server': { url: 'http://localhost' } });
-  });
-
-  it('sets metadata.steps when steps is present in the YAML', () => {
-    const yaml = `${VALID_YAML}steps:\n  - name: step1\n    action: do-something\n`;
-    const parsedWithSteps = AssetParser.parseYaml(yaml, 'workflows/w.yaml')!;
-    const asset = AssetParser.buildAsset(parsedWithSteps, '', 'p', 'url', 'main', 'workflows/w.yaml', 'workflows/w.md');
-    expect(asset.metadata?.steps).toEqual([{ name: 'step1', action: 'do-something' }]);
-  });
-
-  it('leaves metadata undefined when no extra fields are present', () => {
-    const asset = AssetParser.buildAsset(
-      parsed,
-      '',
-      'p',
-      'url',
-      'main',
-      'instructions/my-instruction.yaml',
-      'instructions/my-instruction.md',
-    );
-    expect(asset.metadata).toBeUndefined();
-  });
-
-  it('defaults tags to [] when not specified', () => {
-    const yaml = `
-name: No Tags
-description: A description
-type: instruction
-tools:
-  - claude-code
-`;
-    const parsedNoTags = AssetParser.parseYaml(yaml, 'instructions/no-tags.yaml')!;
-    const asset = AssetParser.buildAsset(parsedNoTags, '', 'p', 'url', 'main', 'instructions/no-tags.yaml', 'instructions/no-tags.md');
-    expect(asset.tags).toEqual([]);
-  });
-
-  it('defaults author to "Unknown" when not specified', () => {
-    const yaml = `
-name: No Author
-description: A description
-type: instruction
-tools:
-  - claude-code
-`;
-    const parsedNoAuthor = AssetParser.parseYaml(yaml, 'instructions/no-author.yaml')!;
-    const asset = AssetParser.buildAsset(parsedNoAuthor, '', 'p', 'url', 'main', 'instructions/no-author.yaml', 'instructions/no-author.md');
-    expect(asset.author).toBe('Unknown');
-  });
-});
-
-describe('AssetParser.parseYaml — schema-as-gatekeeper (arbitrary path discovery)', () => {
-  it('discovers a valid asset at an arbitrary custom directory', () => {
-    const result = AssetParser.parseYaml(VALID_YAML, 'ai-artifacts/my-agent.yaml');
-    expect(result).not.toBeNull();
-    expect(result!.meta.name).toBe('My Instruction');
-    expect(result!.mdPath).toBeUndefined();
-  });
-
-  it('discovers a valid asset at a deeply nested path', () => {
-    const result = AssetParser.parseYaml(VALID_YAML, 'team/ai-assets/copilot/my-agent.yaml');
-    expect(result).not.toBeNull();
-    expect(result!.mdPath).toBeUndefined();
-  });
-
-  it('discovers a valid asset at the repo root (no subdirectory)', () => {
-    const result = AssetParser.parseYaml(VALID_YAML, 'my-instruction.yaml');
-    expect(result).not.toBeNull();
-    expect(result!.mdPath).toBeUndefined();
-  });
-
-  it('rejects a GitHub Actions workflow YAML (missing required fields)', () => {
-    const ghActionsYaml = `
-on:
-  push:
-    branches: [main]
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-`;
-    expect(AssetParser.parseYaml(ghActionsYaml, '.github/workflows/ci.yaml')).toBeNull();
-  });
-
-  it('rejects a Helm values YAML (no name/type/tools)', () => {
-    const helmYaml = `
-replicaCount: 2
-image:
-  repository: nginx
-  tag: latest
-service:
-  type: ClusterIP
-  port: 80
-`;
-    expect(AssetParser.parseYaml(helmYaml, 'helm/values.yaml')).toBeNull();
-  });
-
-  it('rejects a Kubernetes manifest YAML (apiVersion/kind fields, no asset fields)', () => {
-    const k8sYaml = `
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: my-app
-spec:
-  replicas: 1
-`;
-    expect(AssetParser.parseYaml(k8sYaml, 'k8s/deployment.yaml')).toBeNull();
-  });
-
-  it('rejects a YAML with a valid-looking name but invalid type enum', () => {
-    const yaml = `
-name: CI Pipeline
-description: Continuous integration pipeline config
-type: pipeline
-tools:
-  - claude-code
-`;
-    expect(AssetParser.parseYaml(yaml, 'ai-artifacts/pipeline.yaml')).toBeNull();
-  });
-
-  it('rejects a YAML with a valid-looking type but unrecognised tool', () => {
-    const yaml = `
-name: My Agent
-description: An agent config
-type: agent
-tools:
-  - unknown-tool
-`;
-    expect(AssetParser.parseYaml(yaml, 'ai-artifacts/agent.yaml')).toBeNull();
-  });
-
-  it('buildAsset correctly sets yamlPath and mdPath for an arbitrary directory', () => {
-    const parsed = AssetParser.parseYaml(VALID_YAML, 'ai-artifacts/my-agent.yaml')!;
-    const asset = AssetParser.buildAsset(
-      parsed,
-      '# Content',
-      'my-provider',
-      'https://github.com/org/repo',
-      'main',
-      'ai-artifacts/my-agent.yaml',
-      'ai-artifacts/SKILL.md',
-    );
-    expect(asset.yamlPath).toBe('ai-artifacts/my-agent.yaml');
-    expect(asset.mdPath).toBe('ai-artifacts/SKILL.md');
-    expect(asset.id).toBe(AssetParser.buildId('my-provider', 'ai-artifacts/my-agent.yaml'));
-  });
-});