@cyberismo/data-handler 0.0.15 → 0.0.16

package/src/containers/template.ts

@@ -23,9 +23,7 @@ import {
   type Card,
   type CardAttachment,
   CardNameRegEx,
-  type Resource,
 } from '../interfaces/project-interfaces.js';
-import type { CardType, Workflow } from '../interfaces/resource-interfaces.js';
 import { pathExists, stripExtension } from '../utils/file-utils.js';
 import { DefaultContent } from '../resources/create-defaults.js';
 
@@ -42,6 +40,9 @@ import { resourceName } from '../utils/resource-utils.js';
 
 import { ROOT } from '../utils/constants.js';
 
+// @todo: Fix the constructor to not use Resource.
+import type { Resource } from './project/resource-cache.js';
+
 // creates template instance based on a project path and name
 export class Template extends CardContainer {
   private templateName: string;
@@ -55,6 +56,12 @@ export class Template extends CardContainer {
     });
   }
 
+  /**
+   * Creates an instance of Template container that holds template related cards.
+   * @param project Project in which template is.
+   * @param template Template resource that this container is connected to.
+   */
+  // @todo: Fix the constructor to not use Resource, but resource full path
   constructor(project: Project, template: Resource) {
     // Templates might come from modules. Remove module name from template name.
     const templateName = stripExtension(basename(template.name));
@@ -199,18 +206,14 @@ export class Template extends CardContainer {
     // Process metadata
     const processMetadata = async (card: Card, parentCards: Card[]) => {
       if (!card.metadata) return card;
+      const cardType = this.project.resources
+        .byType(card.metadata?.cardType, 'cardTypes')
+        .show();
 
-      const cardType = this.project.resource<CardType>(card.metadata?.cardType);
-      if (!cardType) {
-        throw new Error(
-          `Card type '${card.metadata?.cardType}' of card ${card.key} cannot be found`,
-        );
-      }
+      const workflow = this.project.resources
+        .byType(cardType.workflow, 'workflows')
+        .show();
 
-      const workflow = this.project.resource<Workflow>(cardType.workflow);
-      if (!workflow) {
-        throw new Error(`Workflow '${cardType.workflow}' cannot be found`);
-      }
       const initialWorkflowState = workflow.transitions.find(
         (item) => item.fromState.includes('') || item.fromState.length === 0,
       );
@@ -268,7 +271,7 @@ export class Template extends CardContainer {
           await mkdir(processedCard.path, { recursive: true });
 
           await Promise.all([
-            processedCard.metadata && this.saveCardMetadata(processedCard),
+            this.saveCardMetadata(processedCard),
             writeFile(
               join(processedCard.path, Project.cardContentFile),
               processedAttachments.content || '',
@@ -389,18 +392,19 @@ export class Template extends CardContainer {
     parentCard?: Card,
   ): Promise<string> {
     const destinationCardPath = parentCard
-      ? join(await this.cardFolder(parentCard.key), 'c')
+      ? join(this.cardFolder(parentCard.key), 'c')
       : this.templateCardsPath;
     let newCardKey = '';
 
     try {
+      // todo: to use cache instead of file access
       if (!pathExists(this.templateFolder())) {
         throw new Error(`Template '${this.containerName}' does not exist`);
       }
-      const cardType = this.project.resource<CardType>(cardTypeName);
-      if (cardType === undefined) {
-        throw new Error(`Card type '${cardTypeName}' does not exist`);
-      }
+      const cardType = this.project.resources
+        .byType(cardTypeName, 'cardTypes')
+        .show();
+
       if (parentCard && !this.hasTemplateCard(parentCard.key)) {
         throw new Error(
           `Card '${parentCard.key}' does not exist in template '${this.containerName}'`,
@@ -522,7 +526,7 @@ export class Template extends CardContainer {
   /**
    * Checks if a specific card key exists in a template.
    * @param cardKey Card key to find from template.
-   * @return true if card with a given card key exists in the template, false otherwise.
+   * @returns true if card with a given card key exists in the template, false otherwise.
    */
   public hasTemplateCard(cardKey: string): boolean {
     return this.project.hasTemplateCard(cardKey);
@@ -533,6 +537,7 @@ export class Template extends CardContainer {
    * @returns true, if template is exists in project; false otherwise
    */
   public isCreated(): boolean {
+    // todo: to use cache instead of file access
     return pathExists(this.templateCardsPath);
   }
 

package/src/interfaces/folder-content-interfaces.ts

@@ -13,6 +13,8 @@
 
 import type { Schema } from 'jsonschema';
 
+export type { Schema };
+
 // All file mappings for lookup (filename -> property name)
 export const ALL_FILE_MAPPINGS = {
   'calculation.lp': 'calculation',
@@ -24,7 +26,7 @@ export const ALL_FILE_MAPPINGS = {
 } as const;
 
 // Reverse mappings from property names to filenames
-export const REVERSE_FILE_MAPPINGS = {
+export const CONTENT_FILES = {
   calculation: 'calculation.lp',
   contentTemplate: 'index.adoc.hbs',
   model: 'model.lp',
@@ -34,7 +36,7 @@ export const REVERSE_FILE_MAPPINGS = {
 } as const;
 
 // Union type of all valid content property names
-export type ContentPropertyName = keyof typeof REVERSE_FILE_MAPPINGS;
+export type ContentPropertyName = keyof typeof CONTENT_FILES;
 
 // Content interface for Calculation resources
 export interface CalculationContent {
@@ -49,6 +51,7 @@ export interface GraphModelContent {
 // Content interface for Graph View resources
 export interface GraphViewContent {
   viewTemplate?: string;
+  schema?: Schema;
 }
 
 // Content interface for Report resources
@@ -70,9 +73,7 @@ export type FolderResourceContent =
  * @returns filename that matches property name
  */
 export function filename(propertyName: string): string | undefined {
-  return REVERSE_FILE_MAPPINGS[
-    propertyName as keyof typeof REVERSE_FILE_MAPPINGS
-  ];
+  return CONTENT_FILES[propertyName as keyof typeof CONTENT_FILES];
 }
 
 /**
@@ -80,7 +81,7 @@ export function filename(propertyName: string): string | undefined {
  * @param filename Filename.
  * @returns property name that matches filename
  */
-export function propertyName(
+export function contentPropertyName(
   filename: string,
 ): ContentPropertyName | undefined {
   return ALL_FILE_MAPPINGS[filename as keyof typeof ALL_FILE_MAPPINGS];

package/src/interfaces/project-interfaces.ts

@@ -178,6 +178,7 @@ export interface ProjectMetadata {
 
 // Project's settings (=cardsConfig.json).
 export interface ProjectSettings {
+  schemaVersion?: number;
   cardKeyPrefix: string;
   name: string;
   modules: ModuleSetting[];
@@ -228,12 +229,9 @@ export type RemovableResourceTypes =
   | 'workflow'
   | 'label';
 
-// Project resource, such as workflow, template or card type as file system object.
-export interface Resource {
-  name: string;
-  path: string;
-}
-
+// TODO: fix terminology. In DH, modules are not resources.
+// Also, this contains non-folder resources
+// This was done likely like this, because on the CLI, they act similarly to resources
 // Resources that have own folders.
 export type ResourceFolderType =
   | 'calculations'
@@ -247,6 +245,9 @@ export type ResourceFolderType =
   | 'templates'
   | 'workflows';
 
+// This covers all 'true' resources
+export type ResourceType = Exclude<ResourceFolderType, 'modules'>;
+
 // All resource types; both singular and plural.
 export type ResourceTypes =
   | RemovableResourceTypes

package/src/interfaces/resource-interfaces.ts

@@ -17,6 +17,7 @@ import type {
   GraphViewContent,
   ReportContent,
 } from './folder-content-interfaces.js';
+import type { ResourceType } from './project-interfaces.js';
 
 /**
  * Each resource represents a file (or a folder in some cases) with metadata stored
@@ -140,17 +141,31 @@ export interface ResourceBaseMetadata {
 }
 
 // All resources metadata content.
-export type ResourceContent =
-  | CalculationMetadata
+export type AnyResourceContent =
+  | Calculation
   | CardType
   | FieldType
   | GraphModel
   | GraphView
   | LinkType
-  | ReportMetadata
+  | Report
   | TemplateMetadata
   | Workflow;
 
+export type ResourceContent<T extends ResourceType> = ResourceContentMap[T];
+
+export type ResourceContentMap = {
+  calculations: Calculation;
+  cardTypes: CardType;
+  fieldTypes: FieldType;
+  graphModels: GraphModel;
+  graphViews: GraphView;
+  linkTypes: LinkType;
+  reports: Report;
+  templates: TemplateMetadata;
+  workflows: Workflow;
+};
+
 // Template configuration details.
 export interface TemplateConfiguration extends TemplateMetadata {
   path: string;

package/src/macros/graph/index.ts

@@ -12,19 +12,14 @@
 */
 
 import BaseMacro from '../base-macro.js';
-import type { GraphOptions } from './types.js';
 import { createImage, validateMacroContent } from '../index.js';
 import Handlebars from 'handlebars';
-import type { MacroGenerationContext } from '../../interfaces/macros.js';
 import macroMetadata from './metadata.js';
-import { pathExists } from '../../utils/file-utils.js';
-import { readFile } from 'node:fs/promises';
-import { readFileSync } from 'node:fs';
-import type { Schema } from 'jsonschema';
-import type TaskQueue from '../task-queue.js';
 import { ClingoError } from '@cyberismo/node-clingo';
-import { resourceFilePath } from '../../utils/resource-utils.js';
-import { resourceName } from '../../utils/resource-utils.js';
+
+import type { GraphOptions } from './types.js';
+import type { MacroGenerationContext } from '../../interfaces/macros.js';
+import type TaskQueue from '../task-queue.js';
 
 class GraphMacro extends BaseMacro {
   constructor(tasksQueue: TaskQueue) {
@@ -38,40 +33,35 @@ class GraphMacro extends BaseMacro {
   handleStatic = async (context: MacroGenerationContext, input: unknown) => {
     const options = this.parseOptions(input, context);
 
-    const modelLocation = resourceFilePath(
-      context.project,
-      resourceName(options.model),
-      'model.lp',
-    );
-    const viewLocation = resourceFilePath(
-      context.project,
-      resourceName(options.view),
-      'view.lp.hbs',
+    const modelResource = context.project.resources.byType(
+      options.model,
+      'graphModels',
     );
+    const modelContent = modelResource.contentData();
 
-    if (!pathExists(modelLocation)) {
-      throw new Error(`Graph: Model ${options.model} does not exist`);
+    const viewResource = context.project.resources.byType(
+      options.view,
+      'graphViews',
+    );
+    const viewContent = viewResource.contentData();
+    if (!viewContent.viewTemplate) {
+      throw new Error(`Graph: View ${options.view} has no view template`);
     }
 
-    let viewContent = '';
-    try {
-      viewContent = await readFile(viewLocation, { encoding: 'utf-8' });
-    } catch {
-      throw new Error(`Graph: View ${options.view} does not exist`);
-    }
     const handlebarsContext = {
       cardKey: context.cardKey,
       ...options,
     };
 
     const handlebars = Handlebars.create();
-    const view = handlebars.compile(viewContent)(handlebarsContext);
+    const view = handlebars.compile(viewContent.viewTemplate)(
+      handlebarsContext,
+    );
 
-    const modelContent = await readFile(modelLocation, { encoding: 'utf-8' });
     let result: string;
     try {
       result = await context.project.calculationEngine.runGraph(
-        modelContent,
+        modelContent.model,
         view,
         context.context,
       );
@@ -98,24 +88,13 @@ class GraphMacro extends BaseMacro {
   ): GraphOptions {
     const options = validateMacroContent<GraphOptions>(this.metadata, input);
 
-    let schema: Schema | null = null;
-    try {
-      schema = JSON.parse(
-        readFileSync(
-          resourceFilePath(
-            context.project,
-            resourceName(options.view),
-            'parameterSchema.json',
-          ),
-          { encoding: 'utf-8' },
-        ),
-      );
-    } catch (err) {
-      this.logger.trace(
-        err,
-        'Graph schema not found or failed to read, skipping validation',
-      );
-    }
+    // Get schema from view resource content if available
+    const resource = context.project.resources.byType(
+      options.view,
+      'graphViews',
+    );
+    const content = resource.contentData();
+    const schema = content.schema;
 
     if (schema) {
       validateMacroContent(this.metadata, input, schema);

package/src/macros/index.ts

@@ -186,7 +186,7 @@ export function validateMacroContent<T>(
  * @param tasks - Tasks to register
  * @returns macro instances
  */
-export function registerMacros(
+export async function registerMacros(
   instance: typeof Handlebars,
   context: MacroGenerationContext,
   tasks: TaskQueue,
@@ -242,7 +242,7 @@ export async function evaluateMacros(
 ) {
   const handlebars = Handlebars.create();
   const tasks = new TaskQueue();
-  registerMacros(handlebars, context, tasks);
+  await registerMacros(handlebars, context, tasks);
   let result = content;
   while ((context.maxTries ?? 10) > 0) {
     await tasks.reset();

package/src/macros/report/index.ts

@@ -18,8 +18,6 @@ import macroMetadata from './metadata.js';
 import BaseMacro from '../base-macro.js';
 import { validateJson } from '../../utils/validate.js';
 import type TaskQueue from '../task-queue.js';
-import { ReportResource } from '../../resources/report-resource.js';
-import { resourceName } from '../../utils/resource-utils.js';
 import { generateReportContent } from '../../utils/report.js';
 import { ClingoError } from '@cyberismo/node-clingo';
 import type { ReportOptions } from './types.js';
@@ -34,13 +32,9 @@ class ReportMacro extends BaseMacro {
 
   handleStatic = async (context: MacroGenerationContext, data: unknown) => {
     const options = this.validate(data);
-    const resource = new ReportResource(
-      context.project,
-      resourceName(options.name),
-    );
-    const report = await resource.show();
-
-    if (!report) throw new Error(`Report ${options.name} does not exist`);
+    const report = context.project.resources
+      .byType(options.name, 'reports')
+      .show();
 
     if (report.content.schema) {
       validateJson(options, {

package/src/module-manager.ts

@@ -26,7 +26,7 @@ import { Project } from './containers/project.js';
 import type { ProjectConfiguration } from './project-settings.js';
 import { ProjectPaths } from './containers/project/project-paths.js';
 import { readJsonFile } from './utils/json.js';
-import { Validate } from './commands/index.js';
+import { Validate } from './commands/validate.js';
 
 const FILE_PROTOCOL = 'file:';
 const HTTPS_PROTOCOL = 'https:';
@@ -54,7 +54,7 @@ export class ModuleManager {
     await copyDir(sourcePath, destinationPath);
 
     // Update the resources.
-    await this.project.collectModuleResources();
+    this.project.resources.changedModules();
   }
 
   // Creates a map of what dependencies each module depend from.
@@ -213,13 +213,11 @@ export class ModuleManager {
 
   // Fetches direct dependencies of a module.
   private async dependencies(moduleName: string): Promise<Set<string>> {
-    const allModules = await this.project.modules();
-    if (!allModules) return new Set();
-    const module = allModules.find((m) => m.name === moduleName);
+    const module = await this.project.module(moduleName);
     if (!module) {
       throw new Error(`Module '${moduleName}' not found`);
     }
-    const modulePath = join(module.path, module.name, 'cardsConfig.json');
+    const modulePath = join(module.path, 'cardsConfig.json');
     const moduleConfiguration = (await readJsonFile(
       modulePath,
     )) as ProjectConfiguration;
@@ -287,7 +285,7 @@ export class ModuleManager {
 
   // Imports from a given folder. Is used both for .temp/<module name> and file locations.
   private async importFromFolder(path: string, name: string) {
-    await this.importFileModule(path);
+    await this.importFileModule(path, undefined, true); // Skip validation during updates
     console.log(
       `... Imported module '${name}' to '${this.project.configuration.name}'`,
     );
@@ -361,6 +359,9 @@ export class ModuleManager {
     try {
       await this.removeModuleFiles(module.name);
       console.log(`... Removed imported module '${module.name}'`);
+
+      // Refresh module resources in cache after filesystem removal to avoid stale prefixes
+      this.project.resources.changedModules();
     } catch (error) {
       if (error instanceof Error)
         console.error(
@@ -479,7 +480,11 @@ export class ModuleManager {
   }
 
   // Updates modules in the project.
-  private async update(module?: ModuleSetting, credentials?: Credentials) {
+  private async update(
+    module?: ModuleSetting,
+    credentials?: Credentials,
+    skipModules?: Set<string>,
+  ) {
     // Prints dots every half second so that user knows that something is ongoing
     function start() {
       console.log('... Collecting unique modules. This takes a moment.');
@@ -514,22 +519,46 @@ export class ModuleManager {
         uniqueModules.map((item) => item.name),
       );
 
+      // Filter out modules that are already imported
+      const modulesToImport = skipModules
+        ? uniqueModules.filter((module) => !skipModules.has(module.name))
+        : uniqueModules;
+
+      if (
+        skipModules &&
+        skipModules.size > 0 &&
+        modulesToImport.length < uniqueModules.length
+      ) {
+        const skippedModules = uniqueModules
+          .filter((module) => skipModules.has(module.name))
+          .map((m) => m.name)
+          .join(', ');
+        console.log(
+          `... Skipping already imported module(s): ${skippedModules}`,
+        );
+      }
+
       // Update modules parallel.
       const promises: Promise<void>[] = [];
-      uniqueModules.forEach((module) =>
+      modulesToImport.forEach((module) =>
         promises.push(this.handleModule(module)),
       );
       await Promise.all(promises);
       await deleteDir(this.tempModulesDir);
-      await this.project.collectModuleResources();
+      this.project.resources.changedModules();
     }
   }
 
   // Checks that module prefix is not in use in the project
-  private async validatePrefix(modulePrefix: string) {
+  // Optionally skip check for modules that are already imported (during updates)
+  private validatePrefix(modulePrefix: string, skipIfExists = false) {
     // Do not allow modules with same prefixes.
-    const currentlyUsedPrefixes = await this.project.projectPrefixes();
+    const currentlyUsedPrefixes = this.project.projectPrefixes();
     if (currentlyUsedPrefixes.includes(modulePrefix)) {
+      // If skipIfExists is true, allow re-importing modules that are already present
+      if (skipIfExists) {
+        return;
+      }
       throw new Error(
         `Imported project has a prefix '${modulePrefix}' that is already used in the project. Cannot import from module.`,
       );
@@ -540,8 +569,14 @@ export class ModuleManager {
    * Imports module from local file path.
    * @param source Path to import from.
    * @param destination is this really needed???
+   * @param skipValidation Skip prefix validation (used during updates)
+   * @returns Module prefix of the imported module.
    */
-  public async importFileModule(source: string, destination?: string) {
+  public async importFileModule(
+    source: string,
+    destination?: string,
+    skipValidation = false,
+  ) {
     if (!Validate.validateFolder(source)) {
       throw new Error(
         `Input validation error: folder name is invalid '${source}'`,
@@ -566,7 +601,7 @@ export class ModuleManager {
     );
     const sourcePath = sourceProject.paths.resourcesFolder;
 
-    await this.validatePrefix(modulePrefix);
+    this.validatePrefix(modulePrefix, skipValidation);
 
     // Copy files.
     await this.addFileContents(sourcePath, destinationPath);
@@ -578,12 +613,14 @@ export class ModuleManager {
    * @param source Git URL to import from.
    * @param options Modules setting options.
    * @param credentials Credentials for private repositories.
+   * @param skipValidation Skip prefix validation (used during updates)
    * @returns module prefix as defined in its CardsConfig.json
    */
   public async importGitModule(
     source: string,
     options?: ModuleSettingOptions,
     credentials?: Credentials,
+    skipValidation = false,
   ) {
     const clonedName = await this.clone(
       {
@@ -596,7 +633,7 @@ export class ModuleManager {
     );
     const clonePath = join(this.tempModulesDir, clonedName);
     const modulePrefix = (await this.configuration(clonePath)).cardKeyPrefix;
-    await this.validatePrefix(modulePrefix);
+    this.validatePrefix(modulePrefix, skipValidation);
 
     const sourcePath = new ProjectPaths(clonePath).resourcesFolder;
     const destinationPath = join(
@@ -612,6 +649,7 @@ export class ModuleManager {
    * If module is not used by any other modules, then will remove the module from disk as well.
    * Otherwise, only updates project configuration.
    * @param moduleName Name of the module to remove
+   * @throws If module was not found.
    */
   public async removeModule(moduleName: string) {
     const projectModules = this.project.configuration.modules;
@@ -639,14 +677,33 @@ export class ModuleManager {
     await this.project.removeModule(moduleName);
   }
 
+  /**
+   * Updates dependencies for a module without re-importing the module itself.
+   * Used during module import to fetch dependencies after the main module is already imported.
+   * @param module Module whose dependencies should be updated.
+   * @param credentials Optional credentials for private repositories.
+   * @returns Module prefix as defined in its CardsConfig.json
+   */
+  public async updateDependencies(
+    module: ModuleSetting,
+    credentials?: Credentials,
+  ) {
+    return this.update(module, credentials, new Set([module.name]));
+  }
+
   /**
    * Imports module from a local file path or a git URL.
    * @param module Module to update. If not provided, updates all modules.
    * @param credentials Optional credentials for private repositories.
+   * @param skipModules Optional set of module names to skip during import.
    * @returns Module prefix as defined in its CardsConfig.json
    */
-  public async updateModule(module: ModuleSetting, credentials?: Credentials) {
-    return this.update(module, credentials);
+  public async updateModule(
+    module: ModuleSetting,
+    credentials?: Credentials,
+    skipModules?: Set<string>,
+  ) {
+    return this.update(module, credentials, skipModules);
   }
 
   /**