@intlayer/webpack 1.0.0

package/src/transpiler/intlater_module_to_dictionary/transpileBundledCode.ts

@@ -0,0 +1,107 @@
+import { existsSync } from 'fs';
+import { mkdir, writeFile } from 'fs/promises';
+import { resolve } from 'path';
+import { getConfiguration } from '@intlayer/config';
+import type { ContentModule } from '@intlayer/core';
+import { extractObjectsWithId } from './extractNestedJSON';
+import { processModule } from './processModule';
+
+const { content } = getConfiguration();
+const { dictionariesDir, bundleFileExtension } = content;
+
+const loadBundledModule = async (bundledEntryPath: string) => {
+  const entryFilePath = resolve(bundledEntryPath);
+
+  const isEntryFilePathExist = existsSync(entryFilePath);
+
+  if (!isEntryFilePathExist) {
+    console.error('Entry file not found. ' + entryFilePath);
+    return;
+  }
+
+  const isEntryPathValid = entryFilePath.endsWith(bundleFileExtension);
+
+  if (!isEntryPathValid) {
+    return;
+  }
+
+  try {
+    // Remove the module from the cache
+    delete require.cache[entryFilePath];
+
+    // Require the module anew
+    const entry = await import(entryFilePath);
+
+    let result: ContentModule | undefined;
+
+    if (entry.default) {
+      // JS or TS file
+      result = await processModule(entryFilePath);
+    } else {
+      // JSON file
+      result = entry;
+    }
+
+    return result;
+  } catch (err) {
+    console.error(`Error transpiling ${entryFilePath}:`, err);
+  }
+};
+
+const buildDictionary = async (dictionaries: ContentModule[]) => {
+  const resultDictionariesPaths: string[] = [];
+
+  for await (const content of dictionaries) {
+    const contentString = JSON.stringify(content);
+
+    const id = content.id;
+    const outputFileName = `${id}.json`;
+    const resultFilePath = resolve(dictionariesDir, outputFileName);
+
+    // Create the json file
+    await writeFile(resultFilePath, contentString, 'utf8').catch((err) => {
+      console.error(`Error creating ${outputFileName}:`, err);
+    });
+
+    resultDictionariesPaths.push(resultFilePath);
+  }
+
+  return resultDictionariesPaths;
+};
+
+/**
+ * This function transpile the bundled code to to make dictionaries as JSON files
+ */
+export const transpileBundledCode = async (
+  bundledEntriesPaths: string[] | string
+) => {
+  const resultDictionariesPaths: string[] = [];
+
+  if (typeof bundledEntriesPaths === 'string') {
+    bundledEntriesPaths = [bundledEntriesPaths];
+  }
+
+  const filteredBundledEntriesPaths: string[] = bundledEntriesPaths.filter(
+    // Filter js files;
+    (bundledEntryPath) => bundledEntryPath.endsWith(bundleFileExtension)
+  );
+
+  // Create the dictionaries folder if it doesn't exist
+  await mkdir(resolve(dictionariesDir), { recursive: true });
+
+  for await (const bundledEntryPath of filteredBundledEntriesPaths) {
+    const result = await loadBundledModule(bundledEntryPath);
+
+    if (!result) {
+      continue;
+    }
+
+    const nestedContent: ContentModule[] = extractObjectsWithId(result);
+
+    const dictionariesPaths: string[] = await buildDictionary(nestedContent);
+
+    resultDictionariesPaths.push(...dictionariesPaths);
+  }
+
+  return resultDictionariesPaths;
+};

package/src/utils.ts

@@ -0,0 +1,26 @@
+import { SHA3, enc } from 'crypto-js';
+
+export const getFileHash = (filePath: string) => {
+  const hash = SHA3(filePath);
+
+  return hash
+    .toString(enc.Base64)
+    .replace(/[^A-Z\d]/gi, '')
+    .substring(0, 20);
+};
+
+export const transformToCamelCase = (string: string): string => {
+  // Split the string into words using a regex that finds spaces, hyphens, and underscores
+  const words = string.split(/[\s\-_]+/);
+
+  // Transform each word except the first to have its first letter uppercase
+  const camelCasedWords = words.map((word, index) => {
+    if (index === 0) {
+      return word[0].toUpperCase() + word.slice(1);
+    }
+    return word.charAt(0).toUpperCase() + word.slice(1);
+  });
+
+  // Join the words back together
+  return camelCasedWords.join('');
+};

package/src/webpack-plugin.ts

@@ -0,0 +1,185 @@
+import { relative, resolve } from 'path';
+import { getConfiguration } from '@intlayer/config';
+import { sync } from 'glob';
+import { DynamicEntryPlugin, type Compiler } from 'webpack';
+import { createDictionaryList } from './transpiler/dictionary_to_main/createDictionaryList';
+import { createModuleAugmentation } from './transpiler/dictionary_to_type/createModuleAugmentation';
+import { createTypes } from './transpiler/dictionary_to_type/createType';
+import { transpileBundledCode } from './transpiler/intlater_module_to_dictionary/transpileBundledCode';
+import { getFileHash } from './utils';
+
+const getRelativePath = (filePath: string) => relative(baseDir, filePath);
+
+const getBundledFilePathFromIntlayerModule = (filePath: string): string => {
+  const hash = getFileHash(filePath);
+
+  return `${bundleDir}/${hash}${bundleFileExtension}`;
+};
+
+const { content } = getConfiguration();
+const { bundleDir, baseDir, bundleFileExtension, watchedFilesPatternWithPath } =
+  content;
+
+export class IntLayerPlugin {
+  private managedFiles: Set<string>;
+  private updatedFiles: Set<string>;
+  private addedFiles: Set<string>;
+
+  constructor() {
+    this.managedFiles = new Set();
+    this.updatedFiles = new Set();
+    this.addedFiles = new Set();
+  }
+
+  // function to initialize the dictionaries
+  public async initDictionaries() {
+    const outputFiles = [...this.managedFiles].map((file) =>
+      resolve(bundleDir, file)
+    );
+
+    const dictionaries = (await transpileBundledCode(outputFiles)) ?? [];
+
+    console.info(
+      `Dictionaries: \n ${dictionaries.map(getRelativePath).join('\n')}`
+    );
+
+    console.info('Building TypeScript types...');
+    createTypes(dictionaries);
+
+    console.info('Building type index...');
+    createModuleAugmentation();
+
+    console.info('Building main...');
+    createDictionaryList();
+  }
+
+  // function to process when intlayer module files content are changed
+  public async processFilesChanges() {
+    const dictionaries =
+      (await transpileBundledCode([...this.updatedFiles])) ?? [];
+
+    console.info(
+      `Updated dictionaries: \n ${dictionaries.map(getRelativePath).join('\n')}`
+    );
+
+    console.info('Updating TypeScript types...');
+    createTypes(dictionaries);
+
+    this.updatedFiles.clear();
+  }
+
+  // function to process when new intlayer module is detected
+  public async processNewFiles() {
+    const dictionaries =
+      (await transpileBundledCode([...this.addedFiles])) ?? [];
+
+    console.info(
+      `New dictionaries: \n ${dictionaries.map(getRelativePath).join('\n')}`
+    );
+
+    console.info('Building TypeScript types...');
+    createTypes(dictionaries);
+
+    console.info('Building type index...');
+    createModuleAugmentation();
+
+    console.info('Building main...');
+    createDictionaryList();
+
+    this.managedFiles = new Set([...this.managedFiles, ...this.addedFiles]);
+    this.addedFiles.clear();
+  }
+
+  public async detectFileAddedOrRemoved() {
+    const filesFound: string[] = [];
+
+    for (const pattern of watchedFilesPatternWithPath) {
+      sync(pattern).map((file) => filesFound.push(file));
+    }
+
+    // Detect new files by comparing with files emitted in previous compilation
+    const newFiles = new Set(
+      filesFound.filter((x) => !this.managedFiles.has(x))
+    );
+
+    const removedFiles = new Set(
+      [...this.managedFiles].filter((x) => !filesFound.includes(x))
+    );
+
+    if (
+      // Check if there is new files
+      newFiles.size > 0
+    ) {
+      console.info('New files:', [...newFiles]);
+
+      this.addedFiles = newFiles;
+
+      await this.processNewFiles();
+    }
+
+    if (
+      // Check if there is removed files
+      removedFiles.size > 0
+    ) {
+      console.info('Removed files:', [...removedFiles]);
+
+      this.managedFiles = new Set(
+        [...this.managedFiles].filter((x) => !removedFiles.has(x))
+      );
+    }
+
+    // After the compilation, transpile the changed files if any
+    if (this.updatedFiles.size > 0) {
+      await this.processFilesChanges();
+    }
+  }
+
+  apply(compiler: Compiler): void {
+    compiler.hooks.entryOption.tap('EntryOptionPlugin', (context, entry) => {
+      if (typeof entry === 'function') {
+        // DynamicEntryPlugin is used to add entries at runtime when files are created, updated or deleted
+        new DynamicEntryPlugin(context, entry).apply(compiler);
+      }
+
+      return true;
+    });
+
+    compiler.hooks.afterEmit.tapAsync(
+      'IntLayerPlugin - Process dictionaries',
+      async (compilation, callback) => {
+        // Get a set of files that will be emitted in this compilation
+        const currentEmitFiles = new Set(Object.keys(compilation.assets));
+
+        if (
+          // Check if this first load
+          !this.managedFiles.size
+        ) {
+          // Update previousEmitFiles for the next compilation
+          this.managedFiles = currentEmitFiles;
+
+          await this.initDictionaries();
+        }
+
+        await this.detectFileAddedOrRemoved();
+
+        callback();
+      }
+    );
+
+    // Detect modified files on watch mode
+    compiler.hooks.watchRun.tap(
+      'IntLayerPlugin - Change detection',
+      (compilation) => {
+        if (compilation.modifiedFiles) {
+          const updatedFiles = Array.from(compilation.modifiedFiles);
+
+          const outputFileNames = updatedFiles.map(
+            getBundledFilePathFromIntlayerModule
+          );
+
+          this.updatedFiles = new Set(outputFileNames);
+        }
+      }
+    );
+  }
+}

package/src/webpack.config.ts

@@ -0,0 +1,149 @@
+import { join } from 'path';
+import { getConfiguration } from '@intlayer/config';
+import { sync } from 'glob';
+import type { Configuration as WebPackConfiguration } from 'webpack';
+import type { Configuration as DevServerConfiguration } from 'webpack-dev-server';
+import { getFileHash } from './utils';
+import { IntLayerPlugin } from './webpack-plugin';
+
+const { content } = getConfiguration({
+  verbose: true,
+});
+const { bundleDir, bundleFileExtension, watchedFilesPatternWithPath } = content;
+
+const getEntry = (): Record<string, string> =>
+  sync(watchedFilesPatternWithPath).reduce(
+    (obj, el) => {
+      const hash = getFileHash(el);
+
+      obj[hash] = el;
+      return obj;
+    },
+    {} as Record<string, string>
+  );
+
+// For web interface
+export const devServerConfig: DevServerConfiguration = {
+  // Enable hot module replacement
+  hot: true,
+  // Open the browser
+  open: false,
+  liveReload: false,
+
+  // Enable compression
+  compress: true,
+
+  // History API fallback
+  historyApiFallback: false,
+
+  // Host and port
+  host: 'localhost',
+  port: 8080,
+
+  watchFiles: './src/', // watchedFilesPatternWithPath,
+
+  devMiddleware: {
+    // Enable write to disk to reuse the output
+    writeToDisk: true,
+  },
+
+  // Content base
+  static: {
+    directory: bundleDir,
+  },
+};
+
+export const webpackConfig: WebPackConfiguration = {
+  // Define the environment mode (development or production)
+  mode: 'production', // or 'production'
+  // Entry point of the application
+  target: 'node', // Specifies the target environment
+
+  entry: getEntry,
+  output: {
+    clean: true, // Clean the output directory before emit
+    library: 'IntlLayerContent',
+    libraryTarget: 'umd',
+    filename: `[name]${bundleFileExtension}`,
+    path: bundleDir,
+  },
+
+  cache: false,
+
+  // devtool: 'source-map',
+
+  // stats: {
+  //   preset: 'errors-only',
+  //   warnings: false,
+  // },
+  ignoreWarnings: [/./],
+  resolve: {
+    // Resolve TypeScript, JavaScript and JSON files
+    extensions: ['.ts', '.js', '.json', '.wasm', '.ts', '.tsx', '.mjs', '.cjs'],
+    modules: [
+      // To find the loader module
+      join(__dirname, '..', 'node_modules'), // In the project node_modules
+      join(process.cwd(), 'node_modules'), // In the project node_modules
+      join(process.cwd(), 'node_modules', 'intlayer-cli', 'node_modules'), // Or via another project by importing intlayer
+      join(
+        process.cwd(),
+        'node_modules',
+        'intlayer-cli',
+        'node_modules',
+        '@intlayer/webpack',
+        'node_modules'
+      ), // Or via another project by importing intlayer
+    ],
+  },
+
+  resolveLoader: {
+    // Configure how Webpack finds `loader` modules.
+    modules: [
+      // To find the loader module
+      join(process.cwd(), 'node_modules'), // In the project node_modules
+      join(process.cwd(), 'node_modules', '@intlayer/webpack', 'node_modules'), // Or via another project by importing @intlayer/webpack
+      join(
+        process.cwd(),
+        'node_modules',
+        'intlayer-cli',
+        'node_modules',
+        '@intlayer/webpack',
+        'node_modules'
+      ), // Or via another project by importing intlayer
+    ],
+    // roots: [
+    //   join(process.cwd()), // Project context
+    //   join(process.cwd(), '@intlayer/webpack'), // Or via CLI in another project
+    //   join(process.cwd(), 'intlayer', '@intlayer/webpack'), // Or via CLI in another project
+    // ],
+  },
+
+  module: {
+    rules: [
+      {
+        test: /\.node$/,
+        loader: 'node-loader',
+      },
+      // Use esbuild to compile JavaScript & TypeScript
+      {
+        // Match `.js`, `.jsx`, `.ts` or `.tsx` files
+        test: /\.[jt]sx?$/,
+        loader: 'esbuild-loader',
+        options: {
+          // JavaScript version to compile to
+          target: 'es2015',
+        },
+      },
+
+      // JSON files are supported natively by Webpack 5, no specific loader required
+    ],
+  },
+  devServer: devServerConfig,
+
+  plugins: [
+    new IntLayerPlugin(),
+    // new HotModuleReplacementPlugin()
+  ],
+};
+
+export default webpackConfig;