@podlite/publisher 0.0.2

package/lib/node-utils.js

@@ -0,0 +1,412 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.streamWriteArray = exports.parseSources = exports.processFile = exports.getPublishAttributes = exports.getDocumentAttributes = exports.parseFile = exports.getParserTypeforFile = exports.parseFiles = exports.convertFileLinksToUrl = exports.makeLinksMap = exports.getPathToOpen = void 0;
+const schema_1 = require("@podlite/schema");
+const fs = __importStar(require("fs"));
+const path_1 = __importDefault(require("path"));
+const shared_1 = require("./shared");
+const markdown_1 = require("@podlite/markdown");
+const podlite_1 = require("podlite");
+const gray_matter_1 = __importDefault(require("gray-matter"));
+const glob = require('glob');
+const getPathToOpen = (filepath, parentDocPath) => {
+    const isRemoteReg = new RegExp(/^(https?|ftp):/);
+    const isRemote = isRemoteReg.test(filepath);
+    if (isRemote) {
+        return { isRemote, path: filepath };
+    }
+    const path = require('path');
+    const docDirPath = path.dirname(parentDocPath);
+    return {
+        isRemote,
+        path: path.isAbsolute(filepath) ? filepath : path.normalize(path.join(docDirPath, filepath)),
+    };
+};
+exports.getPathToOpen = getPathToOpen;
+const makeLinksMap = (records) => {
+    const linksMap = {
+        ...Object.fromEntries(records.map(({ publishUrl = '', file }) => [(0, exports.getPathToOpen)(file, './').path, publishUrl])),
+    };
+    return linksMap;
+};
+exports.makeLinksMap = makeLinksMap;
+const convertFileLinksToUrl = (records, additinalMap = {}) => {
+    const linksMap = { ...additinalMap, ...(0, exports.makeLinksMap)(records) };
+    const processed = records.map(item => {
+        const converter = (0, schema_1.makeInterator)({
+            'L<>': (node, ctx, interator) => {
+                const { content, meta } = node;
+                const link = meta ? meta : (0, schema_1.getTextContentFromNode)(content);
+                const r = link.match(/file:\s*(?<path>(.+))\s*$/);
+                const convertFileToUrl = filePath => {
+                    const { isRemote, path } = (0, exports.getPathToOpen)(filePath, item.file);
+                    return isRemote ? null : linksMap[path];
+                };
+                const newLink = r?.groups?.path ? convertFileToUrl(r.groups.path) : link;
+                const newContent = {
+                    type: 'text',
+                    value: newLink,
+                };
+                const updated = meta ? { meta: newLink } : { content: newContent };
+                return { ...node, ...updated };
+            },
+        });
+        const res = converter(item.node, {});
+        // process images inside description
+        let extra = {};
+        // process item description  header and footer
+        const { footer, header, description } = item;
+        if (description) {
+            extra.description = converter(description, {});
+        }
+        if (footer) {
+            extra.footer = converter(footer, {});
+        }
+        if (header) {
+            extra.header = converter(header, {});
+        }
+        return { ...item, node: res, ...extra };
+    });
+    return processed;
+};
+exports.convertFileLinksToUrl = convertFileLinksToUrl;
+function parseFiles(path) {
+    let count = 0;
+    const allFiles = glob
+        .sync(path)
+        .map((f) => {
+        count++;
+        const testData = fs.readFileSync(f).toString();
+        const asAst = (0, shared_1.makeAstFromSrc)(testData);
+        // now check if tree contains block with :pubdate attribute
+        // '* :pubdate'
+        if (!(0, shared_1.isExistsPubdate)(asAst)) {
+            return;
+        }
+        // extract notes
+        const notes = (0, schema_1.getFromTree)(asAst, 'para')
+            .filter(n => (0, schema_1.makeAttrs)(n, {}).exists('pubdate'))
+            .map((n) => {
+            const a_pubdate = (0, schema_1.makeAttrs)(n, {}).getFirstValue('pubdate');
+            // Due to cover some cases whan new Date fail on safari, i.e.
+            // new Date("2022-05-07 10:00:00").getFullYear() -> NaN
+            // convert to ISO 8601 "2022-05-07 10:00:00" -> "2022-05-07T10:00:00"
+            const pubdate = a_pubdate.match(/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}(:\d{2})?$/)
+                ? a_pubdate.replace(' ', 'T')
+                : a_pubdate;
+            return {
+                pubdate,
+                type: 'note',
+                node: n,
+                description: n,
+                file: f,
+            };
+        });
+        const articles = (0, shared_1.getAllArticles)([asAst]).map(({ ...all }) => ({ ...all, file: f }));
+        // note get full posts
+        const pages = (0, schema_1.getFromTree)(asAst, 'pod')
+            .filter(n => (0, schema_1.makeAttrs)(n, {}).exists('pubdate'))
+            .map((n) => {
+            const a_pubdate = (0, schema_1.makeAttrs)(n, {}).getFirstValue('pubdate');
+            // Due to cover some cases whan new Date fail on safari, i.e.
+            // new Date("2022-05-07 10:00:00").getFullYear() -> NaN
+            // convert to ISO 8601 "2022-05-07 10:00:00" -> "2022-05-07T10:00:00"
+            const pubdate = a_pubdate.match(/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}(:\d{2})?$/)
+                ? a_pubdate.replace(' ', 'T')
+                : a_pubdate;
+            return {
+                pubdate,
+                type: 'page',
+                node: n,
+                file: f,
+            };
+        });
+        console.warn(` pages: ${pages.length} articles: ${articles.length}, notes: ${notes.length} from ${f}`);
+        return [...pages, ...articles, ...notes].map(item => {
+            return { ...item, file: f };
+        });
+    })
+        .flat()
+        .filter(Boolean);
+    return allFiles;
+}
+exports.parseFiles = parseFiles;
+const PARSER_TYPES = {
+    MARKDOWN: 'markdown',
+    PODLITE: 'podlite',
+    DEFAULT: 'default',
+    ERROR: 'error',
+};
+const MIME_TYPES_EXTENSIONS = {
+    'text/podlite': ['.podlite', '.pod6'],
+    'text/markdown': ['.md', '.markdown'],
+    'text/plain': ['.txt'],
+    'audio/mpeg': ['.mp3'],
+};
+const PARSER_TYPE_MIME_TYPES = {
+    [PARSER_TYPES.PODLITE]: ['text/podlite'],
+    [PARSER_TYPES.MARKDOWN]: ['text/markdown'],
+    [PARSER_TYPES.DEFAULT]: ['text/plain'],
+    [PARSER_TYPES.ERROR]: ['audio/mpeg'],
+};
+function getParserTypeforFile(filePath, mime) {
+    // get parser type using PARSER_TYPE_MIME_TYPES
+    if (mime) {
+        for (const [key, value] of Object.entries(PARSER_TYPE_MIME_TYPES)) {
+            if (value.includes(mime)) {
+                return key;
+            }
+        }
+        // if not found return default  TODO: possible change to error
+        console.warn(`Mime type ${mime} not found, using default parser`);
+        return PARSER_TYPES.DEFAULT;
+    }
+    const ext = path_1.default.extname(filePath).toLowerCase();
+    const parserTypeMap = {
+        '.md': PARSER_TYPES.MARKDOWN,
+        '.markdown': PARSER_TYPES.MARKDOWN,
+        '.podlite': PARSER_TYPES.PODLITE,
+        '.rakudoc': PARSER_TYPES.PODLITE,
+        '.pod6': PARSER_TYPES.PODLITE,
+    };
+    return parserTypeMap[ext] || PARSER_TYPES.DEFAULT;
+}
+exports.getParserTypeforFile = getParserTypeforFile;
+function parseFile(filePath, fileContent, mime) {
+    // check extension of file and parse it deepnds on mime type
+    const parser_type = getParserTypeforFile(filePath, mime);
+    const typeToParserMap = {
+        [PARSER_TYPES.PODLITE]: (src) => {
+            const podlite_processor = (0, podlite_1.podlite)({ importPlugins: true }).use({});
+            const tree = podlite_processor.parse(src, { skipChain: 0, podMode: 1 });
+            return podlite_processor.toAstResult(tree).interator;
+        },
+        //@ts-ignore TODO: fix this
+        [PARSER_TYPES.MARKDOWN]: (src) => {
+            const { content } = (0, gray_matter_1.default)(src);
+            return (0, markdown_1.parseMd)(content);
+        },
+        [PARSER_TYPES.DEFAULT]: (src) => {
+            const podlite_processor = (0, podlite_1.podlite)({ importPlugins: true }).use({});
+            const tree = podlite_processor.parse(src, { skipChain: 0, podMode: 0 });
+            return podlite_processor.toAstResult(tree).interator;
+        },
+    };
+    const src = fileContent || fs.readFileSync(filePath).toString();
+    return typeToParserMap[parser_type](src);
+}
+exports.parseFile = parseFile;
+function getDocumentAttributes(node) {
+    // filling title
+    let title = '';
+    let description = '';
+    let subtitle;
+    let author;
+    let footer = '';
+    let header = null;
+    (0, schema_1.makeInterator)({
+        NAME: (node, ctx, interator) => {
+            title = (0, schema_1.getTextContentFromNode)(node);
+        },
+        TITLE: (node, ctx, interator) => {
+            title = (0, schema_1.getTextContentFromNode)(node);
+        },
+        DESCRIPTION: (node, ctx, interator) => {
+            description = node.content;
+        },
+        SUBTITLE: (node, ctx, interator) => {
+            subtitle = (0, schema_1.getTextContentFromNode)(node);
+        },
+        AUTHOR: (node, ctx, interator) => {
+            const attr = (0, schema_1.makeAttrs)(node, ctx);
+            author = Object.fromEntries(Object.keys(attr.asHash()).map(k => [k, attr.getFirstValue(k)]));
+        },
+        FOOTER: (node, ctx, interator) => {
+            footer = node.content;
+        },
+        HEADER: (node, ctx, interator) => {
+            header = node.content;
+        },
+    })(node, {});
+    const props = {
+        title,
+        description,
+        subtitle,
+        author,
+        footer,
+        puburl: undefined,
+        pubdate: undefined,
+        ...(header && { header }),
+    };
+    const [podnode] = (0, schema_1.getFromTree)(node.content, 'pod');
+    if (podnode) {
+        // prepare publishUrl
+        const conf = (0, schema_1.makeAttrs)(podnode, {});
+        props.puburl = conf.exists('puburl')
+            ? conf.getFirstValue('puburl')
+            : // check old publishUrl attribute
+                conf.exists('publishUrl')
+                    ? conf.getFirstValue('publishUrl')
+                    : undefined;
+        const a_pubdate = conf.getFirstValue('pubdate');
+        // Due to cover some cases whan new Date fail on safari, i.e.
+        // new Date("2022-05-07 10:00:00").getFullYear() -> NaN
+        // convert to ISO 8601 "2022-05-07 10:00:00" -> "2022-05-07T10:00:00"
+        props.pubdate = a_pubdate?.match(/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}(:\d{2})?$/)
+            ? a_pubdate.replace(' ', 'T')
+            : a_pubdate;
+    }
+    return props;
+}
+exports.getDocumentAttributes = getDocumentAttributes;
+function getPublishAttributes(node) {
+    // filling title
+    let title = '';
+    let description = '';
+    let subtitle;
+    let author;
+    let footer = '';
+    (0, schema_1.makeInterator)({
+        NAME: (node, ctx, interator) => {
+            title = (0, schema_1.getTextContentFromNode)(node);
+        },
+        TITLE: (node, ctx, interator) => {
+            title = (0, schema_1.getTextContentFromNode)(node);
+        },
+        DESCRIPTION: (node, ctx, interator) => {
+            description = node.content;
+        },
+        SUBTITLE: (node, ctx, interator) => {
+            subtitle = (0, schema_1.getTextContentFromNode)(node);
+        },
+        AUTHOR: (node, ctx, interator) => {
+            const attr = (0, schema_1.makeAttrs)(node, ctx);
+            author = Object.fromEntries(Object.keys(attr.asHash()).map(k => [k, attr.getFirstValue(k)]));
+        },
+        FOOTER: (node, ctx, interator) => {
+            footer = node.content;
+        },
+    })(node, {});
+    const props = {
+        title,
+        description,
+        subtitle,
+        author,
+        footer,
+        puburl: undefined,
+        pubdate: undefined,
+    };
+    const podnode = node;
+    if (podnode) {
+        // prepare publishUrl
+        const conf = (0, schema_1.makeAttrs)(podnode, {});
+        props.puburl = conf.exists('puburl')
+            ? conf.getFirstValue('puburl')
+            : // check old publishUrl attribute
+                conf.exists('publishUrl')
+                    ? conf.getFirstValue('publishUrl')
+                    : undefined;
+        const a_pubdate = conf.getFirstValue('pubdate');
+        // Due to cover some cases whan new Date fail on safari, i.e.
+        // new Date("2022-05-07 10:00:00").getFullYear() -> NaN
+        // convert to ISO 8601 "2022-05-07 10:00:00" -> "2022-05-07T10:00:00"
+        props.pubdate = a_pubdate?.match(/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}(:\d{2})?$/)
+            ? a_pubdate.replace(' ', 'T')
+            : a_pubdate;
+    }
+    return props;
+}
+exports.getPublishAttributes = getPublishAttributes;
+function processFile(f, content, mime) {
+    const podlite_document = parseFile(f, content, mime);
+    // now extract some extra meta inforamtion, like pubdate, puburl
+    const attr = ((f, node) => {
+        if (getParserTypeforFile(f, mime) === PARSER_TYPES.MARKDOWN) {
+            // try to extract from markdown front matter
+            const { data } = (0, gray_matter_1.default)(content || fs.readFileSync(f).toString());
+            return data;
+        }
+        else {
+            return getDocumentAttributes(podlite_document);
+        }
+    })(f, podlite_document);
+    // prepare attributes
+    const { title, description, subtitle, author, footer, puburl, pubdate, header } = attr;
+    return {
+        type: 'page',
+        title,
+        description,
+        subtitle,
+        author,
+        ...(header && { header }),
+        footer,
+        publishUrl: puburl,
+        pubdate,
+        file: f,
+        sources: [],
+        node: podlite_document,
+    };
+}
+exports.processFile = processFile;
+function parseSources(path) {
+    let count = 0;
+    const allFiles = glob
+        .sync(path)
+        .map((f) => {
+        count++;
+        return processFile(f);
+    })
+        .flat()
+        .filter(Boolean);
+    return allFiles;
+}
+exports.parseSources = parseSources;
+function streamWriteArray(array, outputPath) {
+    return new Promise((resolve, reject) => {
+        const writeStream = fs.createWriteStream(outputPath);
+        writeStream.write('[');
+        const writeItem = index => {
+            if (index >= array.length) {
+                writeStream.write(']');
+                writeStream.end();
+                resolve(1);
+                return;
+            }
+            const item = array[index];
+            const json = JSON.stringify(item);
+            const data = index === 0 ? json : ',' + json;
+            if (writeStream.write(data)) {
+                setImmediate(() => writeItem(index + 1));
+            }
+            else {
+                writeStream.once('drain', () => writeItem(index + 1));
+            }
+        };
+        writeItem(0);
+        writeStream.on('error', reject);
+    });
+}
+exports.streamWriteArray = streamWriteArray;
+//# sourceMappingURL=node-utils.js.map

package/lib/plugins.d.ts

@@ -0,0 +1,138 @@
+import { PodliteDocument, PodNode } from '@podlite/schema';
+export declare type pubRecord = {
+    type: string;
+    pubdate: string;
+    node: PodNode;
+    description?: PodNode;
+    file: string;
+};
+export declare type publishRecord = pubRecord & {
+    title: string | null;
+    publishUrl: string;
+    sources: string[];
+    node: PodliteDocument;
+    pubdate: string | undefined;
+    template?: publishRecord | null;
+    header?: PodNode | null;
+    footer?: PodNode | null;
+    subtitle?: string | null;
+    pluginsData?: {
+        [name: string]: any;
+    };
+    template_file?: string;
+};
+export interface PodliteWebPluginContext {
+    [name: string]: any;
+}
+export declare type PodliteWebPlugin = [
+    (rects: publishRecord[]) => publishRecord[],
+    (ctx: PodliteWebPluginContext) => {
+        [name: string]: any;
+    }
+];
+export interface PluginConfig {
+    name?: string;
+    plugin: PodliteWebPlugin;
+    includePatterns?: string | string[];
+    excludePatterns?: string | string[];
+}
+/**
+=begin pod
+=head1 filterFiles
+
+This function filters an array of file names based on provided include and exclude patterns. It first converts the includePatterns and excludePatterns arguments into arrays, if they aren't already. The function then filters through the files array, including a file if it matches any of the include patterns (or if no include patterns are provided) and excluding it if it matches any of the exclude patterns. The function returns a new array containing the files that match the include criteria and do not match the exclude criteria.
+
+=head2 Parameters
+
+  =begin item
+  B<files>
+  
+  An array of strings, each representing a file name to be filtered.
+  =end item
+  =begin item
+  B<includePatterns>
+  
+  Optional. A string or an array of strings representing the pattern(s) files must match to be included. If not provided, all files are considered to match the include criteria.
+  =end item
+  =begin item
+  B<excludePatterns>
+  
+  Optional. A string or an array of strings representing the pattern(s) files must match to be excluded. If not provided, no files are excluded based on patterns.
+  =end item
+
+=head2 Returns
+
+An array of strings, each representing a file name that matched the include criteria and did not match the exclude criteria.
+
+=end pod
+*/
+export declare function filterFiles(files: string[], includePatterns?: string | string[], excludePatterns?: string | string[]): string[];
+/**
+=begin pod
+=head1 processPlugin
+
+The function `processPlugin` processes a collection of publishing records based on a given
+plugin configuration. It filters the items to determine which ones match specified inclusion
+and exclusion patterns. Items that match are processed by the provided plugin function, while
+ items that do not match are passed through unchanged. Finally, it returns a tuple containing
+ the processed items and the result of calling the `onClose` function with a context object.
+
+=head2 Parameters
+
+  =begin item
+  B<pluginConf>
+  
+  An object of type `PluginConfig` that provides configuration for the plugin, including
+  the plugin processing function, inclusion patterns, and exclusion patterns.
+  =end item
+  =begin item
+  B<items>
+  
+  An array of `publishRecord` objects representing the items to be processed by the plugin.
+  =end item
+
+=head2 Returns
+
+A tuple where the first element is an array of `publishRecord` objects representing
+the processed and unprocessed items, and the second element is an object resulting
+from calling the plugin's `onClose` function with a context object. This object may
+contain arbitrary data based on the plugin's implementation.
+
+=end pod
+*/
+export declare const processPlugin: (pluginConf: PluginConfig, items: publishRecord[], ctx?: {
+    [name: string]: any;
+}) => [publishRecord[], {
+    [name: string]: any;
+}];
+/**
+=begin pod
+=head1 composePlugins
+
+This function takes an array of PluginConfig objects and combines them into a single PluginConfig.
+It does this by sequentially processing each PluginConfig in the array with the next,
+effectively composing their behaviors into a single plugin configuration.
+Each PluginConfig is expected to modify a shared state and context. The composition
+is achieved by chaining the plugin functions within each PluginConfig, so that the
+output (both state and context) of one plugin function becomes the input to the next.
+
+=head2 Parameters
+
+  =begin item
+  B<configs>
+  
+  An array of PluginConfig objects. Each PluginConfig is an object that represents
+  configuration for a plugin, which includes a plugin processing function and
+  potentially other settings.
+  =end item
+
+=head2 Returns
+
+Returns a single PluginConfig object that represents the composed configuration
+of all the PluginConfig objects passed in the 'configs' array. This resulting
+PluginConfig can be used to process items with the combined logic of all the
+plugins defined in the input array.
+
+=end pod
+*/
+export declare const composePlugins: (configs: PluginConfig[], inintCtx?: {}) => PluginConfig;