erudit 2.0.0-dev.7

package/server/plugin/build/jobs/nav.ts

@@ -0,0 +1,209 @@
+import { globSync } from 'glob';
+import chalk from 'chalk';
+import { resolvePaths } from 'erudit-cog/kit';
+import { contentTypes, topicParts, type ContentType } from 'erudit-cog/schema';
+
+import { PROJECT_DIR } from '#erudit/globalPaths';
+import { stress } from '@erudit/utils/stress';
+import { debug, logger } from '@server/logger';
+import {
+    createRootNode,
+    isRootNode,
+    type NavNode,
+    type RootNavNode,
+} from '@server/nav/node';
+import { ERUDIT_SERVER } from '../../global';
+
+type Ids = Record<string, string>;
+let ids: Ids;
+
+const contentTargets = ERUDIT_SERVER.CONFIG?.contentTargets || [];
+
+const nodePathRegexp = new RegExp(
+    `(?<pos>\\d+)(?<sep>-|\\+)(?<id>[\\w-]+)\\/(?<type>${contentTypes.join('|')})\\..*`,
+);
+
+export async function buildNav() {
+    debug.start('Building navigation tree...');
+
+    ids = {};
+
+    const rootNode = createRootNode();
+    rootNode.children = (await scanChildNodes(rootNode, false)).children;
+    ERUDIT_SERVER.NAV = rootNode.children ? rootNode : undefined;
+
+    const nodeCount = Object.values(ids).length;
+
+    if (nodeCount === 0) {
+        logger.warn('No content nodes detected! The site will be empty!');
+    } else {
+        if (ERUDIT_SERVER.CONFIG?.debug?.log) debugPrintNav(rootNode);
+
+        logger.success(
+            'Navigation tree built successfully!',
+            chalk.dim(`(${nodeCount})`),
+        );
+    }
+}
+
+async function scanChildNodes(
+    parent: NavNode | RootNavNode,
+    insideBook: boolean,
+): Promise<{ children: NavNode[] | undefined; newIds: Ids }> {
+    const currentFsPath = isRootNode(parent) ? '' : parent.path;
+
+    const nodeFsPaths = globSync(
+        `${PROJECT_DIR}/content/${currentFsPath}/*/{${contentTypes.join(',')}}.{ts,js}`,
+    )
+        .sort()
+        .map((path) => resolvePaths(path));
+
+    let newIds: Ids = {};
+    const children: NavNode[] = [];
+
+    for (const nodeFsPath of nodeFsPaths) {
+        const pathParts:
+            | {
+                  pos: string;
+                  sep: string;
+                  id: string;
+                  type: ContentType;
+              }
+            | undefined = nodeFsPath.match(nodePathRegexp)?.groups as any;
+
+        if (!pathParts) continue; // Wrong path pattern
+
+        const nodePath = nodeFsPath
+            .replace(PROJECT_DIR + '/content/', '')
+            .split('/')
+            .slice(0, -1)
+            .join('/');
+
+        if (pathParts.type === 'book' && insideBook) {
+            logger.warn(
+                'Books inside books are not allowed!\n',
+                `Skipping ${stress(nodePath)} nav node!`,
+            );
+            continue;
+        }
+
+        if (!satisfiesContentTargets(nodePath)) continue; // Not a content target
+
+        const parentId = isRootNode(parent)
+            ? ''
+            : parent.skip
+              ? parent.id.split('/').slice(0, -1).join('/')
+              : parent.id;
+
+        // Regular id might not include parent id part if it is skipped
+        const id = parentId ? `${parentId}/${pathParts.id}` : pathParts.id;
+
+        if (ids[id]) {
+            logger.warn(
+                `Nav node ${stress(id)} ID collision!\n\n`,
+                `This ID belongs to ${stress(ids[id], chalk.greenBright)} nav node.\n`,
+                `Nav node ${stress(nodePath, chalk.redBright)} tries to use the same ID!\n`,
+                'Skipping nav node which causes collision!',
+            );
+            continue;
+        }
+
+        // Full id always includes parent id
+        const fullId = isRootNode(parent)
+            ? pathParts.id
+            : `${parent.fullId}/${pathParts.id}`;
+
+        const skip = pathParts.sep === '+';
+
+        const childNode: NavNode = {
+            type: pathParts.type,
+            path: nodePath,
+            id,
+            fullId,
+            skip,
+        };
+
+        if (!validNode(childNode)) continue;
+
+        ids[id] = nodePath;
+        newIds[id] = nodePath;
+
+        const scanResult = await scanChildNodes(
+            childNode,
+            insideBook || childNode.type === 'book',
+        );
+
+        if (
+            ['book', 'group'].includes(childNode.type) &&
+            !scanResult.children
+        ) {
+            delete ids[id];
+            delete newIds[id];
+            for (const childNewId of Object.keys(scanResult.newIds))
+                delete ids[childNewId];
+            continue;
+        }
+
+        if (childNode.type === 'book') {
+            ERUDIT_SERVER.NAV_BOOKS ||= {};
+            ERUDIT_SERVER.NAV_BOOKS[id] = childNode;
+        }
+
+        childNode.children = scanResult.children;
+        children.push(childNode);
+    }
+
+    return {
+        children: children.length > 0 ? children : undefined,
+        newIds,
+    };
+}
+
+function satisfiesContentTargets(nodePath: string): boolean {
+    if (contentTargets.length === 0) return true;
+
+    for (const target of contentTargets)
+        if (nodePath.startsWith(target) || target.search(nodePath) === 0)
+            return true;
+
+    return false;
+}
+
+function validNode(node: NavNode): boolean {
+    switch (node.type) {
+        case 'topic':
+            const partPaths = globSync(
+                PROJECT_DIR +
+                    `/content/${node.path}` +
+                    `/{${topicParts.join(',')}}.bi`,
+            );
+            if (partPaths.length === 0) return false; // Topic is empty
+            break;
+        case 'book':
+            if (node.skip) {
+                logger.warn(
+                    `Books can't skip their ID part!\n`,
+                    `Skipping ${stress(node.path)} nav node!`,
+                );
+                return false;
+            }
+            break;
+    }
+
+    return true;
+}
+
+function debugPrintNav(node: RootNavNode) {
+    const logNode = (node: NavNode | RootNavNode, indent: number) => {
+        console.log(
+            isRootNode(node)
+                ? chalk.dim('#root')
+                : `${'  '.repeat(indent)}${node.id.split('/').pop()} ${chalk.dim(`[${node.type}${node.skip ? ', ' + chalk.yellow('skip') : ''}]`)}`,
+        );
+
+        if (node.children)
+            for (const child of node.children) logNode(child, indent + 1);
+    };
+
+    logNode(node, 0);
+}

package/server/plugin/build/process.ts

@@ -0,0 +1,25 @@
+import { debug, logger } from '@server/logger';
+
+import { setup } from './setup';
+import { setupLanguage } from './jobs/language';
+import { buildContributors } from './jobs/contributors';
+import { buildNav } from './jobs/nav';
+import { buildContent } from './jobs/content/generic';
+
+let initial = true;
+
+export async function build() {
+    debug.start('Building data...');
+
+    if (initial) {
+        await setup();
+        initial = false;
+    }
+
+    await setupLanguage();
+    await buildContributors();
+    await buildNav();
+    await buildContent();
+
+    logger.success('Build successful!');
+}

package/server/plugin/build/rebuild.ts

@@ -0,0 +1,55 @@
+import chokidar, { type FSWatcher } from 'chokidar';
+
+import { ERUDIT_DIR, PROJECT_DIR } from '#erudit/globalPaths';
+import { ERUDIT_SERVER } from '@server/global';
+import { logger } from '@server/logger';
+import { build } from '@server/build/process';
+import { stress } from '@erudit/utils/stress';
+
+const watchTargets: string[] = [
+    // Languages
+    ERUDIT_DIR + '/languages',
+    // Content directory
+    PROJECT_DIR + '/content',
+    // Contributors directory
+    PROJECT_DIR + '/contributors',
+];
+
+const ignoreTargets: string[] = [];
+
+const rebuildDelay = 300;
+
+//
+//
+//
+
+let watcher: FSWatcher;
+let rebuildTimeout: any;
+let rebuildRequested = false;
+
+export async function setupRebuildWatcher() {
+    if (watcher) return;
+
+    watcher = chokidar.watch(watchTargets, {
+        ignored: ignoreTargets,
+        ignoreInitial: true,
+    });
+
+    watcher.on('all', (event, path) =>
+        requestServerRebuild(`File change: ${stress(path)}`),
+    );
+}
+
+export async function requestServerRebuild(reason?: string) {
+    if (rebuildRequested) return;
+
+    clearTimeout(rebuildTimeout);
+    rebuildTimeout = setTimeout(async () => {
+        rebuildRequested = true;
+        await ERUDIT_SERVER.BUILD_PROMISE;
+        console.log('\n');
+        logger.info(`Rebuilding server data!${reason ? ` ${reason}` : ''}\n\n`);
+        await build();
+        rebuildRequested = false;
+    }, rebuildDelay);
+}

package/server/plugin/build/setup.ts

@@ -0,0 +1,21 @@
+import eruditConfig from '#erudit/config';
+
+import { registerGlobals } from '@erudit/globals/register';
+import { debug } from '@server/logger';
+import { ERUDIT_SERVER } from '@server/global';
+
+import { setupRebuildWatcher } from './rebuild';
+import { setupDatabase } from '../db/setup';
+import { setupBitranConfig } from '../bitran/setup';
+
+export async function setup() {
+    debug.start('Running initial setup procedures...');
+
+    if (import.meta.dev) await setupRebuildWatcher();
+
+    registerGlobals();
+    ERUDIT_SERVER.CONFIG = eruditConfig;
+
+    await setupBitranConfig();
+    await setupDatabase();
+}

package/server/plugin/content/absoluteId.ts

@@ -0,0 +1,94 @@
+import { isTopicPart } from 'erudit-cog/schema';
+
+import {
+    parseBitranLocation,
+    stringifyBitranLocation,
+    type BitranLocation,
+} from '@shared/bitran/location';
+import { getNavBookOf } from '../nav/utils';
+
+export function toAbsoluteContentLocation<T extends string | BitranLocation>(
+    location: T,
+    contextContentId: string,
+): T {
+    const isLocationStr = typeof location === 'string';
+    const parsedLocation = isLocationStr
+        ? parseBitranLocation(location)
+        : (location as BitranLocation);
+
+    if (
+        isTopicPart(parsedLocation.type) ||
+        ['book', 'group', 'topic'].includes(parsedLocation.type)
+    ) {
+        parsedLocation.path = toAbsoluteContentId(
+            parsedLocation.path!,
+            contextContentId,
+        );
+        return (
+            isLocationStr
+                ? stringifyBitranLocation(parsedLocation)
+                : parsedLocation
+        ) as T;
+    }
+
+    return location;
+}
+
+export function toAbsoluteContentId(
+    path: string,
+    contextContentId: string = '',
+) {
+    const fullPath = (() => {
+        if (path.startsWith('..'))
+            return (contextContentId ? contextContentId + '/' : '') + path;
+
+        if (path.startsWith('.'))
+            return trimEdgeSlashes(contextContentId + path.substring(1));
+
+        if (path.startsWith('~'))
+            return trimEdgeSlashes(
+                tryGetParentBook(contextContentId) + path.substring(1),
+            );
+
+        return path;
+    })();
+
+    const reversePathParts = fullPath.split('/').reverse();
+    const finalParts: string[] = [];
+
+    let skipCounter = 0;
+
+    for (let i = 0; i < reversePathParts.length; i++) {
+        const part = reversePathParts[i];
+
+        if (part === '..') {
+            skipCounter++;
+            continue;
+        }
+
+        if (skipCounter) {
+            skipCounter--;
+            continue;
+        }
+
+        finalParts.unshift(part!);
+    }
+
+    return finalParts.join('/');
+}
+
+function trimEdgeSlashes(path: string) {
+    if (path.startsWith('/')) path = path.substring(1);
+
+    if (path.endsWith('/')) path = path.substring(0, path.length - 1);
+
+    return path;
+}
+
+function tryGetParentBook(contentId: string): string {
+    const navBook = getNavBookOf(contentId);
+
+    if (!navBook) return '';
+
+    return navBook.id;
+}

package/server/plugin/content/context.ts

@@ -0,0 +1,112 @@
+import { isContentType, isTopicPart, type TopicPart } from 'erudit-cog/schema';
+
+import { ERUDIT_SERVER } from '@server/global';
+import { getIdsUp, isSkipId } from '@server/nav/utils';
+import { DbContent } from '@server/db/entities/Content';
+import { DbContributor } from '@server/db/entities/Contributor';
+
+import type { Context } from '@shared/content/context';
+import {
+    createContentLink,
+    createContributorLink,
+    createTopicPartLink,
+} from '@shared/link';
+import { CONTENT_TYPE_ICON, ICON, TOPIC_PART_ICON } from '@erudit/shared/icons';
+import type { BitranLocation } from '@erudit/shared/bitran/location';
+
+export async function getContentContext(contentId: string): Promise<Context> {
+    const context: Context = [];
+
+    for (const _contentId of (await getIdsUp(contentId)).reverse()) {
+        const dbContent = await getDbContent(_contentId);
+
+        context.push({
+            type: ERUDIT_SERVER.LANGUAGE.phrases[dbContent.type],
+            icon: isTopicPart(dbContent.type)
+                ? TOPIC_PART_ICON[dbContent.type]
+                : isContentType(dbContent.type)
+                  ? CONTENT_TYPE_ICON[dbContent.type]
+                  : '',
+            title: dbContent.title || dbContent.contentId,
+            href: createContentLink(dbContent.type, dbContent.contentId),
+            hidden: await isSkipId(dbContent.contentId),
+        });
+    }
+
+    return context;
+}
+
+export async function getTopicPartContext(
+    topicPart: TopicPart,
+    contentId: string,
+): Promise<Context> {
+    const context = await getContentContext(contentId);
+    const topicItem = context.pop()!;
+
+    topicItem.type = ERUDIT_SERVER.LANGUAGE.phrases[topicPart];
+    topicItem.icon = TOPIC_PART_ICON[topicPart];
+    topicItem.href = createTopicPartLink(topicPart, contentId);
+
+    return context.concat([topicItem]);
+}
+
+export async function getContributorContext(
+    contributorId: string,
+): Promise<Context> {
+    const dbContributor = await ERUDIT_SERVER.DB.manager.findOne(
+        DbContributor,
+        {
+            select: ['displayName'],
+            where: { contributorId },
+        },
+    );
+
+    if (!dbContributor)
+        throw new Error(`Contributor "${contributorId}" not found!`);
+
+    return [
+        {
+            type: ERUDIT_SERVER.LANGUAGE.phrases.contributor,
+            icon: ICON.contributor,
+            title: dbContributor.displayName || contributorId,
+            href: createContributorLink(contributorId),
+            hidden: false,
+        },
+    ];
+}
+
+export async function getLocationContext(
+    location: BitranLocation,
+): Promise<Context> {
+    const contentId = location.path!;
+
+    switch (location.type) {
+        case 'article':
+        case 'summary':
+        case 'practice':
+            return await getTopicPartContext(location.type, contentId);
+        case 'group':
+            return await getContentContext(contentId);
+        case 'contributor':
+            return await getContributorContext(contentId);
+    }
+
+    throw new Error(
+        `Cant' build context for location unknown type "${location.type}"!`,
+    );
+}
+
+//
+//
+//
+
+async function getDbContent(contentId: string): Promise<DbContent> {
+    const dbContent = await ERUDIT_SERVER.DB.manager.findOne(DbContent, {
+        select: ['type', 'title', 'fullId', 'contentId'],
+        where: { contentId },
+    });
+
+    if (!dbContent) throw new Error(`Content "${contentId}" not found!`);
+
+    return dbContent;
+}

package/server/plugin/db/entities/Book.ts

@@ -0,0 +1,7 @@
+import { Entity, PrimaryColumn } from 'typeorm';
+
+@Entity('book')
+export class DbBook {
+    @PrimaryColumn('varchar')
+    contentId!: string;
+}

package/server/plugin/db/entities/Content.ts

@@ -0,0 +1,49 @@
+import { Column, Entity, Index, PrimaryColumn } from 'typeorm';
+import type {
+    ContentFlag,
+    ContentReferences,
+    ContentSeo,
+    ContentType,
+} from 'erudit-cog/schema';
+
+import type { ImageData } from '@erudit/shared/image';
+
+@Entity('content')
+export class DbContent {
+    @PrimaryColumn('varchar')
+    contentId!: string;
+
+    @Column('varchar')
+    @Index({ unique: true })
+    fullId!: string;
+
+    @Column('varchar')
+    type!: ContentType;
+
+    @Column('varchar', { nullable: true })
+    title?: string;
+
+    @Column('varchar', { nullable: true })
+    navTitle?: string;
+
+    @Column('text', { nullable: true })
+    description?: string;
+
+    @Column('simple-json', { nullable: true })
+    flags?: Record<ContentFlag, boolean>;
+
+    @Column('varchar', { nullable: true })
+    decoration?: string;
+
+    @Column('simple-json', { nullable: true })
+    seo?: Partial<ContentSeo>;
+
+    @Column('simple-json', { nullable: true })
+    ogImage?: ImageData;
+
+    @Column('simple-array', { nullable: true })
+    dependencies?: string[];
+
+    @Column('simple-json', { nullable: true })
+    references?: ContentReferences;
+}

package/server/plugin/db/entities/Contribution.ts

@@ -0,0 +1,10 @@
+import { Entity, PrimaryColumn } from 'typeorm';
+
+@Entity('contribution')
+export class DbContribution {
+    @PrimaryColumn('varchar')
+    contentId!: string;
+
+    @PrimaryColumn('varchar')
+    contributorId!: string;
+}

package/server/plugin/db/entities/Contributor.ts

@@ -0,0 +1,16 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+@Entity('contributor')
+export class DbContributor {
+    @PrimaryColumn('varchar')
+    contributorId!: string;
+
+    @Column('varchar', { nullable: true })
+    displayName?: string;
+
+    @Column('text', { nullable: true })
+    description?: string;
+
+    @Column('varchar', { nullable: true })
+    avatar?: string;
+}

package/server/plugin/db/entities/Group.ts

@@ -0,0 +1,14 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+import type { GroupType } from 'erudit-cog/schema';
+
+@Entity('group')
+export class DbGroup {
+    @PrimaryColumn('varchar')
+    contentId!: string;
+
+    @Column('varchar')
+    type!: GroupType;
+
+    @Column('text', { nullable: true })
+    content?: string;
+}

package/server/plugin/db/entities/Hash.ts

@@ -0,0 +1,15 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+/**
+ * TODO:
+ * Create hash of Bitran content of content items and use it to distinguish between content item version.
+ */
+
+@Entity('hash')
+export class DbHash {
+    @PrimaryColumn('varchar')
+    id!: string;
+
+    @Column('varchar')
+    hash!: string;
+}

package/server/plugin/db/entities/Topic.ts

@@ -0,0 +1,20 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+import type { TopicPart } from 'erudit-cog/schema';
+
+@Entity('topic')
+export class DbTopic {
+    @PrimaryColumn('varchar')
+    contentId!: string;
+
+    @Column('simple-array')
+    parts!: TopicPart[];
+
+    @Column('text', { nullable: true })
+    article?: string;
+
+    @Column('text', { nullable: true })
+    summary?: string;
+
+    @Column('text', { nullable: true })
+    practice?: string;
+}

package/server/plugin/db/entities/Unique.ts

@@ -0,0 +1,21 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+import type { BitranContext } from '@erudit/shared/bitran/context';
+
+@Entity('unique')
+export class DbUnique {
+    @PrimaryColumn('varchar')
+    location!: string;
+
+    @Column('varchar')
+    productName!: string;
+
+    @Column('text')
+    content!: string;
+
+    @Column('simple-json')
+    context!: BitranContext;
+
+    @Column('varchar', { nullable: true })
+    title?: string;
+}