@code.store/arcxp-sdk-ts 5.2.0 → 5.3.1

package/dist/index.js

@@ -5,11 +5,12 @@ import fs from 'node:fs';
 import path from 'node:path';
 import FormData from 'form-data';
 import * as ws from 'ws';
+import { TextNode, HTMLElement, CommentNode, parse } from 'node-html-parser';
+import { decode } from 'html-entities';
+import * as xmldoc from 'xmldoc';
 import encode from 'base32-encode';
 import { v5 } from 'uuid';
 import assert from 'node:assert';
-import { TextNode, HTMLElement, CommentNode, parse } from 'node-html-parser';
-import { decode } from 'html-entities';
 
 const safeJSONStringify = (data) => {
     try {
@@ -113,28 +114,6 @@ class ArcAuthor extends ArcAbstractAPI {
     }
 }
 
-class ArcContentOps extends ArcAbstractAPI {
-    constructor(options) {
-        super({ ...options, apiPath: 'contentops/v1' });
-    }
-    async schedulePublish(payload) {
-        const { data } = await this.client.put('/publish', payload);
-        return data;
-    }
-    async scheduleUnpublish(payload) {
-        const { data } = await this.client.put('/unpublish', payload);
-        return data;
-    }
-    async unscheduleUnpublish(payload) {
-        const { data } = await this.client.put('/unschedule_unpublish', payload);
-        return data;
-    }
-    async unschedulePublish(payload) {
-        const { data } = await this.client.put('/unschedule_publish', payload);
-        return data;
-    }
-}
-
 class ArcContent extends ArcAbstractAPI {
     constructor(options) {
         super({ ...options, apiPath: 'content/v4' });
@@ -159,6 +138,28 @@ class ArcContent extends ArcAbstractAPI {
     }
 }
 
+class ArcContentOps extends ArcAbstractAPI {
+    constructor(options) {
+        super({ ...options, apiPath: 'contentops/v1' });
+    }
+    async schedulePublish(payload) {
+        const { data } = await this.client.put('/publish', payload);
+        return data;
+    }
+    async scheduleUnpublish(payload) {
+        const { data } = await this.client.put('/unpublish', payload);
+        return data;
+    }
+    async unscheduleUnpublish(payload) {
+        const { data } = await this.client.put('/unschedule_unpublish', payload);
+        return data;
+    }
+    async unschedulePublish(payload) {
+        const { data } = await this.client.put('/unschedule_publish', payload);
+        return data;
+    }
+}
+
 class Custom extends ArcAbstractAPI {
     constructor(options) {
         super({ ...options, apiPath: '' });
@@ -172,6 +173,128 @@ class Custom extends ArcAbstractAPI {
     }
 }
 
+class ArcDeveloperRetail extends ArcAbstractAPI {
+    constructor(options) {
+        super({ ...options, apiPath: 'retail/api/v1' });
+    }
+    // ============================================
+    // Product Methods
+    // ============================================
+    async getProductById(id, params) {
+        const { data } = await this.client.get(`/product/${id}`, { params });
+        return data;
+    }
+    async getProductBySku(sku, params) {
+        const { data } = await this.client.get(`/product/sku/${sku}`, { params });
+        return data;
+    }
+    async getProductByPriceCode(priceCode, params) {
+        const { data } = await this.client.get(`/product/pricecode/${priceCode}`, { params });
+        return data;
+    }
+    async getAllProducts(params) {
+        const { data } = await this.client.get('/product', { params });
+        return data;
+    }
+    // ============================================
+    // Pricing Strategy Methods
+    // ============================================
+    async getPricingStrategyById(id, params) {
+        const { data } = await this.client.get(`/pricing/strategy/${id}`, { params });
+        return data;
+    }
+    async getAllPricingStrategies(params) {
+        const { data } = await this.client.get('/pricing/strategy', { params });
+        return data;
+    }
+    // ============================================
+    // Pricing Rate Methods
+    // ============================================
+    async getPricingRateById(id, params) {
+        const { data } = await this.client.get(`/pricing/rate/${id}`, { params });
+        return data;
+    }
+    async getAllPricingRates(params) {
+        const { data } = await this.client.get('/pricing/rate', { params });
+        return data;
+    }
+    // ============================================
+    // Pricing Cycle Methods
+    // ============================================
+    async getPricingCycle(priceCode, cycleIndex, startDate, params) {
+        const { data } = await this.client.get(`/pricing/cycle/${priceCode}/${cycleIndex}/${startDate}`, {
+            params,
+        });
+        return data;
+    }
+    // ============================================
+    // Campaign Methods
+    // ============================================
+    async getCampaignById(id, params) {
+        const { data } = await this.client.get(`/campaign/${id}`, { params });
+        return data;
+    }
+    async getCampaignByName(campaignName, params) {
+        const { data } = await this.client.get(`/campaign/${campaignName}/get`, { params });
+        return data;
+    }
+    async getAllCampaigns(params) {
+        const { data } = await this.client.get('/campaign', { params });
+        return data;
+    }
+    // ============================================
+    // Campaign Category Methods
+    // ============================================
+    async getCampaignCategoryById(id, params) {
+        const { data } = await this.client.get(`/campaign/category/${id}`, { params });
+        return data;
+    }
+    async getAllCampaignCategories(params) {
+        const { data } = await this.client.get('/campaign/category', { params });
+        return data;
+    }
+    // ============================================
+    // Offer Methods
+    // ============================================
+    async getOfferById(id, params) {
+        const { data } = await this.client.get(`/offer/${id}`, { params });
+        return data;
+    }
+    async getAllOffers(params) {
+        const { data } = await this.client.get('/offer', { params });
+        return data;
+    }
+    // ============================================
+    // Offer Attribute Methods
+    // ============================================
+    async getOfferAttributeById(id, params) {
+        const { data } = await this.client.get(`/offer/attribute/${id}`, { params });
+        return data;
+    }
+    async getAllOfferAttributes(params) {
+        const { data } = await this.client.get('/offer/attribute', { params });
+        return data;
+    }
+    // ============================================
+    // Product Attribute Methods
+    // ============================================
+    async getProductAttributeById(id, params) {
+        const { data } = await this.client.get(`/product/attribute/${id}`, { params });
+        return data;
+    }
+    async getAllProductAttributes(params) {
+        const { data } = await this.client.get('/product/attribute', { params });
+        return data;
+    }
+    // ============================================
+    // Condition Category Methods
+    // ============================================
+    async getAllConditionCategories(params) {
+        const { data } = await this.client.get('/condition/categories', { params });
+        return data;
+    }
+}
+
 class ArcDraft extends ArcAbstractAPI {
     constructor(options) {
         super({ ...options, apiPath: 'draft/v1' });
@@ -568,190 +691,68 @@ class ArcRetailEvents {
     }
 }
 
-class ArcDeveloperRetail extends ArcAbstractAPI {
+class ArcSales extends ArcAbstractAPI {
     constructor(options) {
-        super({ ...options, apiPath: 'retail/api/v1' });
+        super({ ...options, apiPath: 'sales/api/v1' });
     }
-    // ============================================
-    // Product Methods
-    // ============================================
-    async getProductById(id, params) {
-        const { data } = await this.client.get(`/product/${id}`, { params });
+    async migrate(params, payload) {
+        const FormData = await platform.form_data();
+        const form = new FormData();
+        form.append('file', JSON.stringify(payload), { filename: 'subs.json', contentType: 'application/json' });
+        const { data } = await this.client.post('/migrate', form, {
+            params,
+            headers: {
+                ...form.getHeaders(),
+            },
+        });
         return data;
     }
-    async getProductBySku(sku, params) {
-        const { data } = await this.client.get(`/product/sku/${sku}`, { params });
-        return data;
+}
+class ArcSalesV2 extends ArcAbstractAPI {
+    constructor(options) {
+        super({ ...options, apiPath: 'sales/api/v2' });
     }
-    async getProductByPriceCode(priceCode, params) {
-        const { data } = await this.client.get(`/product/pricecode/${priceCode}`, { params });
+    async getEnterpriseGroups(params) {
+        const { data } = await this.client.get('/subscriptions/enterprise', {
+            params: {
+                'arc-site': params.site,
+            },
+        });
         return data;
     }
-    async getAllProducts(params) {
-        const { data } = await this.client.get('/product', { params });
+    async createEnterpriseGroup(params, payload) {
+        const { data } = await this.client.post('/subscriptions/enterprise', payload, {
+            params: {
+                'arc-site': params.site,
+            },
+        });
         return data;
     }
-    // ============================================
-    // Pricing Strategy Methods
-    // ============================================
-    async getPricingStrategyById(id, params) {
-        const { data } = await this.client.get(`/pricing/strategy/${id}`, { params });
+    async createNonce(website, enterpriseGroupId) {
+        const { data } = await this.client.get(`/subscriptions/enterprise/${enterpriseGroupId}`, {
+            params: { 'arc-site': website },
+        });
         return data;
     }
-    async getAllPricingStrategies(params) {
-        const { data } = await this.client.get('/pricing/strategy', { params });
-        return data;
+}
+
+class ArcSigningService extends ArcAbstractAPI {
+    constructor(options) {
+        super({ ...options, apiPath: 'signing-service' });
     }
-    // ============================================
-    // Pricing Rate Methods
-    // ============================================
-    async getPricingRateById(id, params) {
-        const { data } = await this.client.get(`/pricing/rate/${id}`, { params });
+    async sign(service, serviceVersion, imageId) {
+        const { data } = await this.client.get(`/v2/sign/${service}/${serviceVersion}?value=${encodeURI(imageId)}`);
         return data;
     }
-    async getAllPricingRates(params) {
-        const { data } = await this.client.get('/pricing/rate', { params });
-        return data;
+}
+
+class ArcSite extends ArcAbstractAPI {
+    constructor(options) {
+        super({ ...options, apiPath: 'site/v3' });
     }
-    // ============================================
-    // Pricing Cycle Methods
-    // ============================================
-    async getPricingCycle(priceCode, cycleIndex, startDate, params) {
-        const { data } = await this.client.get(`/pricing/cycle/${priceCode}/${cycleIndex}/${startDate}`, {
-            params,
-        });
-        return data;
-    }
-    // ============================================
-    // Campaign Methods
-    // ============================================
-    async getCampaignById(id, params) {
-        const { data } = await this.client.get(`/campaign/${id}`, { params });
-        return data;
-    }
-    async getCampaignByName(campaignName, params) {
-        const { data } = await this.client.get(`/campaign/${campaignName}/get`, { params });
-        return data;
-    }
-    async getAllCampaigns(params) {
-        const { data } = await this.client.get('/campaign', { params });
-        return data;
-    }
-    // ============================================
-    // Campaign Category Methods
-    // ============================================
-    async getCampaignCategoryById(id, params) {
-        const { data } = await this.client.get(`/campaign/category/${id}`, { params });
-        return data;
-    }
-    async getAllCampaignCategories(params) {
-        const { data } = await this.client.get('/campaign/category', { params });
-        return data;
-    }
-    // ============================================
-    // Offer Methods
-    // ============================================
-    async getOfferById(id, params) {
-        const { data } = await this.client.get(`/offer/${id}`, { params });
-        return data;
-    }
-    async getAllOffers(params) {
-        const { data } = await this.client.get('/offer', { params });
-        return data;
-    }
-    // ============================================
-    // Offer Attribute Methods
-    // ============================================
-    async getOfferAttributeById(id, params) {
-        const { data } = await this.client.get(`/offer/attribute/${id}`, { params });
-        return data;
-    }
-    async getAllOfferAttributes(params) {
-        const { data } = await this.client.get('/offer/attribute', { params });
-        return data;
-    }
-    // ============================================
-    // Product Attribute Methods
-    // ============================================
-    async getProductAttributeById(id, params) {
-        const { data } = await this.client.get(`/product/attribute/${id}`, { params });
-        return data;
-    }
-    async getAllProductAttributes(params) {
-        const { data } = await this.client.get('/product/attribute', { params });
-        return data;
-    }
-    // ============================================
-    // Condition Category Methods
-    // ============================================
-    async getAllConditionCategories(params) {
-        const { data } = await this.client.get('/condition/categories', { params });
-        return data;
-    }
-}
-
-class ArcSales extends ArcAbstractAPI {
-    constructor(options) {
-        super({ ...options, apiPath: 'sales/api/v1' });
-    }
-    async migrate(params, payload) {
-        const FormData = await platform.form_data();
-        const form = new FormData();
-        form.append('file', JSON.stringify(payload), { filename: 'subs.json', contentType: 'application/json' });
-        const { data } = await this.client.post('/migrate', form, {
-            params,
-            headers: {
-                ...form.getHeaders(),
-            },
-        });
-        return data;
-    }
-}
-class ArcSalesV2 extends ArcAbstractAPI {
-    constructor(options) {
-        super({ ...options, apiPath: 'sales/api/v2' });
-    }
-    async getEnterpriseGroups(params) {
-        const { data } = await this.client.get('/subscriptions/enterprise', {
-            params: {
-                'arc-site': params.site,
-            },
-        });
-        return data;
-    }
-    async createEnterpriseGroup(params, payload) {
-        const { data } = await this.client.post('/subscriptions/enterprise', payload, {
-            params: {
-                'arc-site': params.site,
-            },
-        });
-        return data;
-    }
-    async createNonce(website, enterpriseGroupId) {
-        const { data } = await this.client.get(`/subscriptions/enterprise/${enterpriseGroupId}`, {
-            params: { 'arc-site': website },
-        });
-        return data;
-    }
-}
-
-class ArcSigningService extends ArcAbstractAPI {
-    constructor(options) {
-        super({ ...options, apiPath: 'signing-service' });
-    }
-    async sign(service, serviceVersion, imageId) {
-        const { data } = await this.client.get(`/v2/sign/${service}/${serviceVersion}?value=${encodeURI(imageId)}`);
-        return data;
-    }
-}
-
-class ArcSite extends ArcAbstractAPI {
-    constructor(options) {
-        super({ ...options, apiPath: 'site/v3' });
-    }
-    async getSections(params) {
-        const { data } = await this.client.get(`/website/${params.website}/section`, {
-            params: { _website: params.website, ...params },
+    async getSections(params) {
+        const { data } = await this.client.get(`/website/${params.website}/section`, {
+            params: { _website: params.website, ...params },
         });
         return data;
     }
@@ -889,85 +890,7 @@ const ArcAPI = (options) => {
     return API;
 };
 
-/* eslint-disable */
-/**
- * This file was automatically generated by json-schema-to-typescript.
- * DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file,
- * and run json-schema-to-typescript to regenerate this file.
- */
-
-var ansTypes = /*#__PURE__*/Object.freeze({
-    __proto__: null
-});
-
-var utils = /*#__PURE__*/Object.freeze({
-    __proto__: null
-});
-
-var ANSType;
-(function (ANSType) {
-    ANSType["Story"] = "story";
-    ANSType["Video"] = "video";
-    ANSType["Tag"] = "tag";
-    ANSType["Author"] = "author";
-    ANSType["Gallery"] = "gallery";
-    ANSType["Image"] = "image";
-    ANSType["Redirect"] = "redirect";
-})(ANSType || (ANSType = {}));
-var MigrationStatus;
-(function (MigrationStatus) {
-    MigrationStatus["Success"] = "Success";
-    MigrationStatus["Queued"] = "Queued";
-    MigrationStatus["Circulated"] = "Circulated";
-    MigrationStatus["Published"] = "Published";
-    MigrationStatus["Scheduled"] = "Scheduled";
-    MigrationStatus["FailVideo"] = "FailVideo";
-    MigrationStatus["FailImage"] = "FailImage";
-    MigrationStatus["FailPhoto"] = "FailPhoto";
-    MigrationStatus["FailStory"] = "FailStory";
-    MigrationStatus["FailGallery"] = "FailGallery";
-    MigrationStatus["FailAuthor"] = "FailAuthor";
-    MigrationStatus["FailTag"] = "FailTag";
-    MigrationStatus["ValidationFailed"] = "ValidationFailed";
-})(MigrationStatus || (MigrationStatus = {}));
-var SummarySortBy;
-(function (SummarySortBy) {
-    SummarySortBy["CreateDate"] = "createDate";
-    SummarySortBy["UpdateDate"] = "updateDate";
-    SummarySortBy["Id"] = "id";
-})(SummarySortBy || (SummarySortBy = {}));
-var SummarySortOrder;
-(function (SummarySortOrder) {
-    SummarySortOrder["ASC"] = "ASC";
-    SummarySortOrder["DESC"] = "DESC";
-})(SummarySortOrder || (SummarySortOrder = {}));
-
-var index$3 = /*#__PURE__*/Object.freeze({
-    __proto__: null,
-    ANS: ansTypes,
-    get ANSType () { return ANSType; },
-    get MigrationStatus () { return MigrationStatus; },
-    get SummarySortBy () { return SummarySortBy; },
-    get SummarySortOrder () { return SummarySortOrder; },
-    TypeUtils: utils
-});
-
-const reference = (ref) => {
-    return {
-        _id: ref.id,
-        type: 'reference',
-        referent: {
-            ...ref,
-        },
-    };
-};
-
-var ANS = /*#__PURE__*/Object.freeze({
-    __proto__: null,
-    reference: reference
-});
-
-const ContentElement = {
+const ContentElement$1 = {
     divider: () => {
         return {
             type: 'divider',
@@ -1199,18 +1122,60 @@ const ContentElement = {
     },
 };
 
+const BLOCK_ELEMENT_TAGS$1 = [
+    'ADDRESS',
+    'ARTICLE',
+    'ASIDE',
+    'BLOCKQUOTE',
+    'DETAILS',
+    'DIV',
+    'DL',
+    'FIELDSET',
+    'FIGCAPTION',
+    'FIGURE',
+    'FOOTER',
+    'FORM',
+    'H1',
+    'H2',
+    'H3',
+    'H4',
+    'H5',
+    'H6',
+    'HEADER',
+    'HR',
+    'LINE',
+    'MAIN',
+    'MENU',
+    'NAV',
+    'OL',
+    'P',
+    'PARAGRAPH',
+    'PRE',
+    'SECTION',
+    'TABLE',
+    'UL',
+    'LI',
+    'BODY',
+    'HTML',
+];
+
+var html_constants = /*#__PURE__*/Object.freeze({
+    __proto__: null,
+    BLOCK_ELEMENT_TAGS: BLOCK_ELEMENT_TAGS$1
+});
+
 const socialRegExps = {
-    instagram: /(?:https?:\/\/)?(?:www.)?instagram.com\/?([a-zA-Z0-9\.\_\-]+)?\/([p]+)?([reel]+)?([tv]+)?([stories]+)?\/([a-zA-Z0-9\-\_\.]+)\/?([0-9]+)?/,
-    twitter: /https:\/\/(?:www\.)?twitter\.com\/[^\/]+\/status(?:es)?\/(\d+)/,
-    tiktok: /https:\/\/(?:m|www|vm)?\.?tiktok\.com\/((?:.*\b(?:(?:usr|v|embed|user|video)\/|\?shareId=|\&item_id=)(\d+))|\w+)/,
-    facebookPost: /https:\/\/www\.facebook\.com\/(photo(\.php|s)|permalink\.php|media|questions|notes|[^\/]+\/(activity|posts))[\/?].*$/,
-    facebookVideo: /https:\/\/www\.facebook\.com\/([^\/?].+\/)?video(s|\.php)[\/?].*/,
+    instagram: /(?:https?:\/\/)?(?:www.)?instagram.com\/?([a-zA-Z0-9._-]+)?\/([p]+)?([reel]+)?([tv]+)?([stories]+)?\/([a-zA-Z0-9\-_.]+)\/?([0-9]+)?/,
+    twitter: /https:\/\/(?:www\.)?twitter\.com\/[^/]+\/status(?:es)?\/(\d+)/,
+    tiktok: /https:\/\/(?:m|www|vm)?\.?tiktok\.com\/((?:.*\b(?:(?:usr|v|embed|user|video)\/|\?shareId=|&item_id=)(\d+))|\w+)/,
+    facebookPost: /https:\/\/www\.facebook\.com\/(photo(\.php|s)|permalink\.php|media|questions|notes|[^/]+\/(activity|posts))[/?].*$/,
+    facebookVideo: /https:\/\/www\.facebook\.com\/([^/?].+\/)?video(s|\.php)[/?].*/,
 };
 function match(url, regex) {
     return url.match(regex)?.[0];
 }
 function youtubeURLParser(url = '') {
-    const regExp = /(?:youtube(?:-nocookie)?\.com\/(?:[^\/\n\s]+\/\S+\/|(?:v|e(?:mbed)?)\/|\S*?[?&]vi?=)|youtu\.be\/)([a-zA-Z0-9_-]{11})/;
+    const regExp = /(?:youtube(?:-nocookie)?\.com\/(?:[^/\n\s]+\/\S+\/|(?:v|e(?:mbed)?)\/|\S*?[?&]vi?=)|youtu\.be\/)([a-zA-Z0-9_-]{11})/;
     const id = url?.match(regExp)?.[1];
     if (id) {
         return `https://youtu.be/${id}`;
@@ -1240,273 +1205,698 @@ function createSocial(url = '') {
     const embeds = [];
     const instagram = instagramURLParser(url);
     if (instagram) {
-        embeds.push(ContentElement.instagram(instagram));
+        embeds.push(ContentElement$1.instagram(instagram));
     }
     const twitter = twitterURLParser(url);
     if (twitter) {
-        embeds.push(ContentElement.twitter(twitter));
+        embeds.push(ContentElement$1.twitter(twitter));
     }
     const tiktok = tiktokURLParser(url);
     if (tiktok) {
-        embeds.push(ContentElement.tiktok(tiktok));
+        embeds.push(ContentElement$1.tiktok(tiktok));
     }
     const youtube = youtubeURLParser(url);
     if (youtube) {
-        embeds.push(ContentElement.youtube(youtube));
+        embeds.push(ContentElement$1.youtube(youtube));
     }
     const facebookPost = facebookPostURLParser(url);
     if (facebookPost) {
-        embeds.push(ContentElement.facebook_post(facebookPost));
+        embeds.push(ContentElement$1.facebook_post(facebookPost));
     }
     const facebookVideo = facebookVideoURLParser(url);
     if (facebookVideo) {
-        embeds.push(ContentElement.facebook_video(facebookVideo));
+        embeds.push(ContentElement$1.facebook_video(facebookVideo));
     }
     return embeds;
 }
 const randomId = () => `${new Date().toISOString()}-${Math.random()}`;
+const isTextCE = (ce) => {
+    return ce?.type === 'text';
+};
+const decodeHTMLEntities = (str) => decode(str);
 
 var ContentElements = /*#__PURE__*/Object.freeze({
     __proto__: null,
     createSocial: createSocial,
+    decodeHTMLEntities: decodeHTMLEntities,
     facebookPostURLParser: facebookPostURLParser,
     facebookVideoURLParser: facebookVideoURLParser,
     instagramURLParser: instagramURLParser,
+    isTextCE: isTextCE,
     randomId: randomId,
     tiktokURLParser: tiktokURLParser,
     twitterURLParser: twitterURLParser,
     youtubeURLParser: youtubeURLParser
 });
 
-const generateArcId = (identifier, orgHostname) => {
-    const namespace = v5(orgHostname, v5.DNS);
-    const buffer = v5(identifier, namespace, Buffer.alloc(16));
-    return encode(buffer, 'RFC4648', { padding: false });
+const isTextNode$1 = (node) => {
+    return node instanceof TextNode;
 };
-/**
- * Utility class for generating Arc IDs and source IDs
- *
- * @example
- * ```ts
- * const generator = new IdGenerator(['my-org']);
- * const arcId = generator.getArcId('123'); // Generates a unique for 'my-org' Arc ID
- * const sourceId = generator.getSourceId('123', ['my-site']); // Generates 'my-site-123'
- * ```
- */
-class IdGenerator {
-    constructor(namespaces) {
-        if (!namespaces.length) {
-            throw new Error('At least 1 namespace is required');
-        }
-        this.namespace = namespaces.join('-');
-    }
-    getArcId(id) {
-        return generateArcId(id.toString(), this.namespace);
-    }
-    getSourceId(id, prefixes = []) {
-        return [...prefixes, id].join('-');
-    }
-}
-
-var Id = /*#__PURE__*/Object.freeze({
-    __proto__: null,
-    IdGenerator: IdGenerator,
-    generateArcId: generateArcId
-});
-
-const buildTree = (items) => {
-    const tree = [
-        {
-            id: '/',
-            children: [],
-            meta: new Proxy({}, {
-                get: () => {
-                    throw new Error('Root node meta is not accessible');
-                },
-            }),
-            parent: null,
-        },
-    ];
-    // Track nodes at each level to maintain parent-child relationships
-    // stores last node at each level
-    const currLevelNodes = {
-        0: tree[0],
-    };
-    for (const item of items) {
-        const node = {
-            id: item.id,
-            parent: null,
-            children: [],
-            meta: item,
-        };
-        // Determine the level of this node
-        const levelKey = Object.keys(item).find((key) => key.startsWith('N') && item[key]);
-        const level = Number(levelKey?.replace('N', '')) || 0;
-        if (!level) {
-            throw new Error(`Invalid level for section ${item.id}`);
-        }
-        // This is a child node - attach to its parent
-        const parentLevel = level - 1;
-        const parentNode = currLevelNodes[parentLevel];
-        if (parentNode) {
-            node.parent = parentNode;
-            parentNode.children.push(node);
-        }
-        else {
-            throw new Error(`Parent node not found for section ${item.id}`);
-        }
-        // Set this as the current node for its level
-        currLevelNodes[level] = node;
-    }
-    // return root nodes children
-    return tree[0].children;
+const isHTMLElement = (node) => {
+    return node instanceof HTMLElement;
 };
-const flattenTree = (tree) => {
-    const flatten = [];
-    const traverse = (node) => {
-        flatten.push(node);
-        for (const child of node.children) {
-            traverse(child);
-        }
-    };
-    // traverse all root nodes and their children
-    for (const node of tree) {
-        traverse(node);
-    }
-    return flatten;
+const isCommentNode = (node) => {
+    return node instanceof CommentNode;
 };
-const buildAndFlattenTree = (items) => flattenTree(buildTree(items));
-const groupByWebsites = (sections) => {
-    return sections.reduce((acc, section) => {
-        const website = section._website;
-        if (!acc[website])
-            acc[website] = [];
-        acc[website].push(section);
-        return acc;
-    }, {});
+const nodeTagIs = (node, name) => {
+    return isHTMLElement(node) && node.tagName?.toLowerCase() === name.toLowerCase();
 };
-const references = (sections) => {
-    return sections.map((s) => reference({
-        id: s._id,
-        website: s._website,
-        type: 'section',
-    }));
+const nodeTagIn = (node, names) => {
+    return isHTMLElement(node) && names.includes(node.tagName?.toLowerCase());
 };
-const isReference = (section) => {
-    return section?.type === 'reference' && section?.referent?.type === 'section';
+const htmlToText = (html, parseOptions) => {
+    if (!html)
+        return '';
+    const doc = parse(html, parseOptions);
+    return decodeHTMLEntities(doc.innerText);
 };
-const removeDuplicates = (sections) => {
-    const map = new Map();
-    sections.forEach((s) => {
-        if (isReference(s)) {
-            map.set(`${s.referent.id}${s.referent.website}`, s);
-        }
-        else {
-            map.set(`${s._id}${s._website}`, s);
-        }
-    });
-    return [...map.values()];
+const getHTMLElementAttribute = (e, key) => {
+    const value = e.getAttribute(key);
+    if (value)
+        return value;
+    return new URLSearchParams(e.rawAttrs.replaceAll(' ', '&')).get(key);
 };
-class SectionsRepository {
-    constructor(arc) {
-        this.arc = arc;
-        this.sectionsByWebsite = {};
-        this.websitesAreLoaded = false;
-    }
-    async put(ans) {
-        await this.arc.Site.putSection(ans);
-        const created = await this.arc.Site.getSection(ans._id, ans.website);
-        this.save(created);
-    }
-    async loadWebsite(website) {
-        const sections = [];
-        let next = true;
-        let offset = 0;
-        while (next) {
-            const migrated = await this.arc.Site.getSections({ website, offset }).catch((_) => {
-                return { q_results: [] };
-            });
-            if (migrated.q_results.length) {
-                sections.push(...migrated.q_results);
-                offset += migrated.q_results.length;
-            }
-            else {
-                next = false;
-            }
-        }
-        return sections;
-    }
-    async loadWebsites(websites) {
-        for (const website of websites) {
-            this.sectionsByWebsite[website] = await this.loadWebsite(website);
-        }
-        this.websitesAreLoaded = true;
-    }
-    save(section) {
-        const website = section._website;
-        assert.ok(website, 'Section must have a website');
-        this.sectionsByWebsite[website] = this.sectionsByWebsite[website] || [];
-        if (!this.sectionsByWebsite[website].find((s) => s._id === section._id)) {
-            this.sectionsByWebsite[website].push(section);
-        }
-    }
-    getById(id, website) {
-        this.ensureWebsitesLoaded();
-        const section = this.sectionsByWebsite[website]?.find((s) => s._id === id);
-        return section;
-    }
-    getByWebsite(website) {
-        this.ensureWebsitesLoaded();
-        return this.sectionsByWebsite[website];
-    }
-    getParentSections(section) {
-        this.ensureWebsitesLoaded();
-        const parents = [];
-        let current = section;
-        while (current.parent?.default && current.parent.default !== '/') {
-            const parent = this.getById(current.parent.default, section._website);
-            if (!parent)
-                break;
-            parents.push(parent);
-            current = parent;
-        }
-        return parents;
-    }
-    ensureWebsitesLoaded() {
-        assert.ok(this.websitesAreLoaded, 'call .loadWebsites() first');
-    }
-}
 
-var Section = /*#__PURE__*/Object.freeze({
+var html_utils = /*#__PURE__*/Object.freeze({
     __proto__: null,
-    SectionsRepository: SectionsRepository,
-    buildAndFlattenTree: buildAndFlattenTree,
-    buildTree: buildTree,
-    flattenTree: flattenTree,
-    groupByWebsites: groupByWebsites,
-    isReference: isReference,
-    references: references,
-    removeDuplicates: removeDuplicates
+    getHTMLElementAttribute: getHTMLElementAttribute,
+    htmlToText: htmlToText,
+    isCommentNode: isCommentNode,
+    isHTMLElement: isHTMLElement,
+    isTextNode: isTextNode$1,
+    nodeTagIn: nodeTagIn,
+    nodeTagIs: nodeTagIs
 });
 
-const ArcUtils = {
-    Id,
-    ANS,
-    ContentElements,
-    Section,
-};
-
 /**
- * Base class for all arc entities, it provides common methods and properties
- * If you want to create a new entity subtype you should extend this class
+ * HTMLProcessor is responsible for parsing HTML content into structured content elements.
+ * It provides a flexible way to handle different HTML nodes and wrap text content.
  *
- * Use case: You want to migrate stories from BBC
- * You define `class BBCStory extends ArcDocument<ANS.AStory>` and implement all abstract methods
- * Then you can override the specific methods to enrich the story with the data from BBC
+ * The processor can be extended with custom handlers for specific node types and
+ * wrappers for text content.
  *
- * To migrate it call .migrate() method
+ * @example
+ * ```ts
+ * // Create and initialize processor
+ * const processor = new HTMLProcessor();
+ * processor.init();
+ *
+ * // Parse HTML content
+ * const html = '<div><p>Some text</p><img src="image.jpg"></div>';
+ * const elements = await processor.parse(html);
+ * ```
+ *
+ * The processor comes with built-in handlers for common HTML elements like links,
+ * text formatting (i, u, strong), and block elements. Custom handlers can be added
+ * using the `handle()` and `wrap()` methods.
  */
-class Document {
+class HTMLProcessor {
+    constructor() {
+        this.parallelProcessing = true;
+        this.handlers = {
+            node: new Map(),
+            wrap: new Map(),
+        };
+    }
+    init() {
+        // wrappers are used to wrap the content of nested text nodes
+        // in a specific way
+        this.wrap('link', (node, text) => {
+            if (nodeTagIn(node, ['a'])) {
+                const attributes = ['href', 'target', 'rel']
+                    .map((attr) => [attr, getHTMLElementAttribute(node, attr)])
+                    .filter(([_, value]) => value)
+                    .map(([key, value]) => `${key}="${value}"`)
+                    .join(' ');
+                return {
+                    ...text,
+                    content: `<a ${attributes}>${text.content}</a>`,
+                };
+            }
+        });
+        this.wrap('i', (node, text) => {
+            if (nodeTagIn(node, ['i'])) {
+                return {
+                    ...text,
+                    content: `<i>${text.content}</i>`,
+                };
+            }
+        });
+        this.wrap('u', (node, text) => {
+            if (nodeTagIn(node, ['u'])) {
+                return {
+                    ...text,
+                    content: `<u>${text.content}</u>`,
+                };
+            }
+        });
+        this.wrap('sup/sub', (node, text) => {
+            if (nodeTagIn(node, ['sup', 'sub'])) {
+                return {
+                    ...text,
+                    content: `<mark class="${node.tagName.toLowerCase()}">${text.content}</mark>`,
+                };
+            }
+        });
+        this.wrap('strong', (node, text) => {
+            if (nodeTagIn(node, ['strong', 'b'])) {
+                return {
+                    ...text,
+                    content: `<b>${text.content}</b>`,
+                };
+            }
+        });
+        this.wrap('center', (node, text) => {
+            if (nodeTagIn(node, ['center'])) {
+                return {
+                    ...text,
+                    alignment: 'center',
+                };
+            }
+        });
+        this.wrap('aligned-paragraph', (node, text) => {
+            if (nodeTagIn(node, ['p'])) {
+                const styleAttribute = getHTMLElementAttribute(node, 'style') || '';
+                if (!styleAttribute)
+                    return text;
+                if (styleAttribute.includes('text-align: right;')) {
+                    return {
+                        ...text,
+                        alignment: 'right',
+                    };
+                }
+                if (styleAttribute.includes('text-align: left;')) {
+                    return {
+                        ...text,
+                        alignment: 'left',
+                    };
+                }
+                if (styleAttribute.includes('text-align: center;')) {
+                    return {
+                        ...text,
+                        alignment: 'center',
+                    };
+                }
+                return text;
+            }
+        });
+        // handlers are used to handle specific nodes
+        // and return a list of content elements
+        this.handle('default', (node) => {
+            const noTag = isHTMLElement(node) && !node.tagName;
+            if (noTag ||
+                nodeTagIn(node, [
+                    'p',
+                    'a',
+                    'b',
+                    'sup',
+                    'sub',
+                    'span',
+                    'strong',
+                    'em',
+                    'i',
+                    'u',
+                    'section',
+                    'main',
+                    'div',
+                    'li',
+                    'center',
+                ])) {
+                return this.handleNested(node);
+            }
+        });
+        this.handle('headers', (node) => {
+            if (nodeTagIn(node, ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'])) {
+                return this.createHeader(node);
+            }
+        });
+        this.handle('text', (node) => {
+            if (isTextNode$1(node)) {
+                return this.createText(node);
+            }
+        });
+        this.handle('comment', (node) => {
+            if (isCommentNode(node)) {
+                return this.handleComment(node);
+            }
+        });
+        this.handle('list', async (node) => {
+            if (nodeTagIn(node, ['ul', 'ol'])) {
+                const listType = node.tagName === 'UL' ? 'unordered' : 'ordered';
+                return this.createList(node, listType);
+            }
+        });
+        this.handle('table', (node) => {
+            if (nodeTagIs(node, 'table')) {
+                return this.handleTable(node);
+            }
+        });
+        this.handle('iframe', (node) => {
+            if (nodeTagIs(node, 'iframe')) {
+                return this.handleIframe(node);
+            }
+        });
+        this.handle('img', (node) => {
+            if (nodeTagIs(node, 'img')) {
+                return this.handleImage(node);
+            }
+        });
+        this.handle('br', (node) => {
+            if (nodeTagIs(node, 'br')) {
+                return this.handleBreak(node);
+            }
+        });
+    }
+    handle(name, handler) {
+        if (this.handlers.node.has(name)) {
+            this.warn({ name }, `${name} node handler already set`);
+        }
+        this.handlers.node.set(name, handler);
+    }
+    wrap(name, handler) {
+        if (this.handlers.wrap.has(name)) {
+            this.warn({ name }, `${name} wrap handler already set`);
+        }
+        this.handlers.wrap.set(name, handler);
+    }
+    async parse(html) {
+        const doc = parse(html, { comment: true });
+        doc.removeWhitespace();
+        const elements = await this.process(doc);
+        const filtered = elements?.filter((e) => e.type !== 'divider');
+        return filtered || [];
+    }
+    addTextAdditionalProperties(c, parent) {
+        const additionalProperties = c.additional_properties || {};
+        const parentNodeIsBlockElement = this.isBlockElement(parent);
+        c.additional_properties = {
+            ...c.additional_properties,
+            isBlockElement: additionalProperties.isBlockElement || parentNodeIsBlockElement,
+        };
+        return c;
+    }
+    /**
+     * Wraps text content elements with additional properties and handlers.
+     * This method iterates through an array of content elements and applies
+     * wrappers to text elements.
+     *
+     * @param node - The HTML node containing the text elements
+     **/
+    wrapChildrenTextNodes(node, elements) {
+        const wrapped = [];
+        const wrappers = [...this.handlers.wrap.values()];
+        for (const c of elements) {
+            if (!isTextCE(c)) {
+                wrapped.push(c);
+                continue;
+            }
+            this.addTextAdditionalProperties(c, node);
+            const handled = wrappers.map((wrapper) => wrapper(node, c)).find(Boolean);
+            wrapped.push(handled || c);
+        }
+        return wrapped;
+    }
+    /**
+     * Handles nested nodes by processing their children and merging text elements.
+     * This method recursively processes the children of a given HTML node and
+     * returns a list of content elements.
+     *
+     * @param node - The HTML node to process
+     **/
+    async handleNested(node) {
+        const children = await this.processChildNodes(node);
+        const filtered = children.filter(Boolean).flat();
+        const merged = this.mergeParagraphs(filtered);
+        const wrapped = this.wrapChildrenTextNodes(node, merged);
+        return wrapped;
+    }
+    async processChildNodes(node) {
+        if (this.parallelProcessing) {
+            return await Promise.all(node.childNodes.map((child) => this.process(child)));
+        }
+        const children = [];
+        for (const child of node.childNodes) {
+            children.push(await this.process(child));
+        }
+        return children;
+    }
+    /**
+     * Processes a single HTML node and converts it into content elements.
+     * This method iterates through registered node handlers and attempts to process the node.
+     * If a handler successfully processes the node, it returns an array of content elements.
+     *
+     * @param node - The HTML node to process
+     * @returns Promise resolving to an array of content elements, or undefined if node cannot be processed
+     */
+    async process(node) {
+        let isKnownNode = false;
+        const elements = [];
+        for (const [name, handler] of this.handlers.node.entries()) {
+            try {
+                const result = await handler(node);
+                if (result) {
+                    // if handler returns an array of elements, it means that the node was handled properly, even if there is no elements inside
+                    isKnownNode = true;
+                    elements.push(...result);
+                    break;
+                }
+            }
+            catch (error) {
+                this.warn({ node: node.toString(), error: error.toString(), name }, 'HandlerError');
+            }
+        }
+        if (isKnownNode)
+            return elements;
+        this.warn({ node: node.toString() }, 'UnknownNodeError');
+    }
+    /**
+     * Merges adjacent text content elements into a single paragraph.
+     * This method iterates through an array of content elements and combines
+     * adjacent text elements into a single paragraph.
+     *
+     * @param items - The array of content elements to merge
+     **/
+    mergeParagraphs(items) {
+        const merged = [];
+        let toMerge = [];
+        const merge = () => {
+            if (!toMerge.length)
+                return;
+            const paragraph = toMerge.reduce((acc, p) => {
+                return {
+                    ...p,
+                    content: acc.content + p.content,
+                };
+            }, { type: 'text', content: '' });
+            merged.push(paragraph);
+            toMerge = [];
+        };
+        for (let i = 0; i < items.length; i++) {
+            const item = items[i];
+            const isBlockElement = item.additional_properties?.isBlockElement;
+            if (isTextCE(item) && !isBlockElement) {
+                toMerge.push(item);
+            }
+            else {
+                merge();
+                merged.push(item);
+            }
+        }
+        merge();
+        return merged;
+    }
+    handleComment(_) {
+        return [];
+    }
+    async handleTable(node) {
+        return [ContentElement$1.raw_html(node.toString())];
+    }
+    async handleIframe(node) {
+        return [ContentElement$1.raw_html(node.toString())];
+    }
+    async handleImage(node) {
+        return [ContentElement$1.raw_html(node.toString())];
+    }
+    async handleBreak(_) {
+        return [ContentElement$1.divider()];
+    }
+    async createQuote(node) {
+        const items = await this.handleNested(node);
+        return [ContentElement$1.quote(items)];
+    }
+    async createText(node) {
+        const text = ContentElement$1.text(node.text);
+        return [text];
+    }
+    filterListItems(items) {
+        return items.filter((i) => ['text', 'list'].includes(i.type));
+    }
+    async createList(node, type) {
+        const items = await this.handleNested(node);
+        return [ContentElement$1.list(type, this.filterListItems(items))];
+    }
+    async createHeader(node) {
+        const level = +node.tagName.split('H')[1] || 3;
+        return [ContentElement$1.header(node.innerText, level)];
+    }
+    isBlockElement(node) {
+        if (!isHTMLElement(node))
+            return false;
+        const defaultBlockElements = new Set(BLOCK_ELEMENT_TAGS$1);
+        return defaultBlockElements.has(node.tagName);
+    }
+    warn(metadata, message) {
+        console.warn(metadata, message);
+    }
+}
+
+var index$4 = /*#__PURE__*/Object.freeze({
+    __proto__: null,
+    Constants: html_constants,
+    HTMLProcessor: HTMLProcessor,
+    Utils: html_utils
+});
+
+const BLOCK_ELEMENT_TAGS = ['paragraph', 'line', 'header', 'ul', 'ol', 'li', 'embed', 'iframe', 'table'];
+
+var xml_constants = /*#__PURE__*/Object.freeze({
+    __proto__: null,
+    BLOCK_ELEMENT_TAGS: BLOCK_ELEMENT_TAGS
+});
+
+const isXmlElement = (node) => {
+    return node?.type === 'element';
+};
+const isTextNode = (node) => {
+    return node?.type === 'text';
+};
+const nodeNameIs = (node, name) => {
+    return isXmlElement(node) && node.name === name;
+};
+const nodeNameIn = (node, names) => {
+    return isXmlElement(node) && names.includes(node.name);
+};
+
+var xml_utils = /*#__PURE__*/Object.freeze({
+    __proto__: null,
+    isTextNode: isTextNode,
+    isXmlElement: isXmlElement,
+    nodeNameIn: nodeNameIn,
+    nodeNameIs: nodeNameIs
+});
+
+const ContentElement = ContentElement$1;
+class XMLProcessor {
+    constructor() {
+        this.handlers = {
+            node: new Map(),
+            wrap: new Map(),
+        };
+    }
+    init() {
+        // wrappers are used to wrap the content of nested text nodes
+        // in a specific way
+        this.wrap('link', (node, content) => {
+            return `<a href="${node.attr.url || node.attr.href || '/'}">${content}</a>`;
+        });
+        this.wrap('header', (_node, content) => {
+            return `<h3>${content}</h3>`;
+        });
+        this.wrap('emphasize', (_node, content) => {
+            return `<i>${content}</i>`;
+        });
+        this.wrap('strong', (_node, content) => {
+            return `<b>${content}</b>`;
+        });
+        // handlers are used to handle specific nodes
+        // and return a list of content elements
+        this.handle('default', (node) => {
+            if (nodeNameIn(node, ['section', 'paragraph', 'line', 'header', 'emphasize', 'strong', 'link', 'li'])) {
+                return this.handleNested(node);
+            }
+        });
+        this.handle('text', (node) => {
+            if (isTextNode(node)) {
+                return [ContentElement.text(node.text)];
+            }
+        });
+        this.handle('list', async (node) => {
+            if (nodeNameIn(node, ['ul', 'ol'])) {
+                const listType = node.name === 'ul' ? 'unordered' : 'ordered';
+                return this.createList(node, listType);
+            }
+        });
+        this.handle('table', (node) => {
+            if (nodeNameIs(node, 'table')) {
+                return this.handleTable(node);
+            }
+        });
+    }
+    async parse(xml) {
+        const doc = new xmldoc.XmlDocument(xml);
+        const elements = await this.process(doc);
+        return elements || [];
+    }
+    handle(name, handler) {
+        if (this.handlers.node.has(name)) {
+            throw new Error(`${name} node handler already set`);
+        }
+        this.handlers.node.set(name, handler);
+    }
+    wrap(name, handler) {
+        if (this.handlers.wrap.has(name)) {
+            throw new Error(`${name} wrap handler already set`);
+        }
+        this.handlers.wrap.set(name, handler);
+    }
+    addTextAdditionalProperties(c, parent) {
+        const additionalProperties = c.additional_properties || {};
+        const parentNodeIsBlockElement = this.isBlockElement(parent);
+        c.additional_properties = {
+            ...c.additional_properties,
+            isBlockElement: additionalProperties.isBlockElement || parentNodeIsBlockElement,
+        };
+        return c;
+    }
+    wrapChildrenTextNodes(node, elements) {
+        const wrapped = [];
+        for (const c of elements) {
+            if (!isTextCE(c)) {
+                wrapped.push(c);
+                continue;
+            }
+            this.addTextAdditionalProperties(c, node);
+            const handler = this.handlers.wrap.get(node.name);
+            if (handler) {
+                wrapped.push({
+                    ...c,
+                    content: handler(node, c.content),
+                });
+            }
+            else {
+                wrapped.push(c);
+            }
+        }
+        return wrapped;
+    }
+    async handleNested(node) {
+        const children = await Promise.all(node.children.map((child) => this.process(child)));
+        const filtered = children.filter(Boolean).flat();
+        const merged = this.mergeParagraphs(filtered);
+        const wrapped = this.wrapChildrenTextNodes(node, merged);
+        return wrapped;
+    }
+    async process(node) {
+        let isKnownNode = false;
+        const elements = [];
+        for (const [name, handler] of this.handlers.node.entries()) {
+            try {
+                const result = await handler(node);
+                if (Array.isArray(result)) {
+                    // if handler returns an array of elements, it means that the node was handled properly, even if there is no elements inside
+                    isKnownNode = true;
+                    elements.push(...result);
+                    break;
+                }
+            }
+            catch (error) {
+                this.warn({ node: node.toString(), error: error.toString(), name }, 'HandlerError');
+            }
+        }
+        if (isKnownNode)
+            return elements;
+        this.warn({ node: node.toString(), type: node.type }, 'UnknownNodeError');
+    }
+    mergeParagraphs(items) {
+        const merged = [];
+        let toMerge = [];
+        const merge = () => {
+            if (!toMerge.length)
+                return;
+            const paragraph = toMerge.reduce((acc, p) => {
+                return {
+                    ...p,
+                    content: acc.content + p.content,
+                };
+            }, { type: 'text', content: '' });
+            merged.push(paragraph);
+            toMerge = [];
+        };
+        for (let i = 0; i < items.length; i++) {
+            const item = items[i];
+            const isBlockElement = item.additional_properties?.isBlockElement;
+            if (isTextCE(item) && !isBlockElement) {
+                toMerge.push(item);
+            }
+            else {
+                merge();
+                merged.push(item);
+            }
+        }
+        merge();
+        return merged;
+    }
+    async handleTable(node) {
+        const html = node.toString({ html: true });
+        return [ContentElement.raw_html(html)];
+    }
+    async createQuote(node) {
+        const items = await this.handleNested(node);
+        return [ContentElement.quote(items)];
+    }
+    async createList(node, type) {
+        const items = await this.handleNested(node);
+        return [ContentElement.list(type, items)];
+    }
+    getNodeInnerText(node) {
+        return node.children.map((n) => this.htmlFromNode(n).innerText.trim());
+    }
+    getNodeInnerHTML(node) {
+        return node.children.map((n) => this.htmlFromNode(n).innerHTML.trim());
+    }
+    htmlFromNode(node) {
+        return parse(node.toString({ html: true }));
+    }
+    getDecodedHTMLFromInnerNodes(node) {
+        const encodedHtml = this.getNodeInnerText(node).join('');
+        const decoded = decodeHTMLEntities(encodedHtml);
+        return decoded;
+    }
+    isBlockElement(node) {
+        const defaultBlockElements = new Set(BLOCK_ELEMENT_TAGS);
+        if (defaultBlockElements.has(node.name))
+            return true;
+    }
+    warn(metadata, message) {
+        console.warn(metadata, message);
+    }
+}
+
+var index$3 = /*#__PURE__*/Object.freeze({
+    __proto__: null,
+    Constants: xml_constants,
+    Utils: xml_utils,
+    XMLProcessor: XMLProcessor
+});
+
+var index$2 = /*#__PURE__*/Object.freeze({
+    __proto__: null,
+    ContentElement: ContentElement$1,
+    HTML: index$4,
+    XML: index$3
+});
+
+/**
+ * Base class for all arc entities, it provides common methods and properties
+ * If you want to create a new entity subtype you should extend this class
+ *
+ * Use case: You want to migrate stories from BBC
+ * You define `class BBCStory extends ArcDocument<ANS.AStory>` and implement all abstract methods
+ * Then you can override the specific methods to enrich the story with the data from BBC
+ *
+ * To migrate it call .migrate() method
+ */
+class Document {
     constructor() {
         this.ans = null;
         this.circulations = [];
@@ -1744,473 +2134,307 @@ class Story extends Document {
     }
 }
 
-var index$2 = /*#__PURE__*/Object.freeze({
+var index$1 = /*#__PURE__*/Object.freeze({
     __proto__: null,
     Document: Document,
     Story: Story
 });
 
-const BLOCK_ELEMENT_TAGS = [
-    'ADDRESS',
-    'ARTICLE',
-    'ASIDE',
-    'BLOCKQUOTE',
-    'DETAILS',
-    'DIV',
-    'DL',
-    'FIELDSET',
-    'FIGCAPTION',
-    'FIGURE',
-    'FOOTER',
-    'FORM',
-    'H1',
-    'H2',
-    'H3',
-    'H4',
-    'H5',
-    'H6',
-    'HEADER',
-    'HR',
-    'LINE',
-    'MAIN',
-    'MENU',
-    'NAV',
-    'OL',
-    'P',
-    'PARAGRAPH',
-    'PRE',
-    'SECTION',
-    'TABLE',
-    'UL',
-    'LI',
-    'BODY',
-    'HTML',
-];
+var ANSType;
+(function (ANSType) {
+    ANSType["Story"] = "story";
+    ANSType["Video"] = "video";
+    ANSType["Tag"] = "tag";
+    ANSType["Author"] = "author";
+    ANSType["Gallery"] = "gallery";
+    ANSType["Image"] = "image";
+    ANSType["Redirect"] = "redirect";
+})(ANSType || (ANSType = {}));
+var MigrationStatus;
+(function (MigrationStatus) {
+    MigrationStatus["Success"] = "Success";
+    MigrationStatus["Queued"] = "Queued";
+    MigrationStatus["Circulated"] = "Circulated";
+    MigrationStatus["Published"] = "Published";
+    MigrationStatus["Scheduled"] = "Scheduled";
+    MigrationStatus["FailVideo"] = "FailVideo";
+    MigrationStatus["FailImage"] = "FailImage";
+    MigrationStatus["FailPhoto"] = "FailPhoto";
+    MigrationStatus["FailStory"] = "FailStory";
+    MigrationStatus["FailGallery"] = "FailGallery";
+    MigrationStatus["FailAuthor"] = "FailAuthor";
+    MigrationStatus["FailTag"] = "FailTag";
+    MigrationStatus["ValidationFailed"] = "ValidationFailed";
+})(MigrationStatus || (MigrationStatus = {}));
+var SummarySortBy;
+(function (SummarySortBy) {
+    SummarySortBy["CreateDate"] = "createDate";
+    SummarySortBy["UpdateDate"] = "updateDate";
+    SummarySortBy["Id"] = "id";
+})(SummarySortBy || (SummarySortBy = {}));
+var SummarySortOrder;
+(function (SummarySortOrder) {
+    SummarySortOrder["ASC"] = "ASC";
+    SummarySortOrder["DESC"] = "DESC";
+})(SummarySortOrder || (SummarySortOrder = {}));
 
-var html_constants = /*#__PURE__*/Object.freeze({
+/* eslint-disable */
+/**
+ * This file was automatically generated by json-schema-to-typescript.
+ * DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file,
+ * and run json-schema-to-typescript to regenerate this file.
+ */
+
+var ansTypes = /*#__PURE__*/Object.freeze({
+    __proto__: null
+});
+
+var utils = /*#__PURE__*/Object.freeze({
+    __proto__: null
+});
+
+var index = /*#__PURE__*/Object.freeze({
     __proto__: null,
-    BLOCK_ELEMENT_TAGS: BLOCK_ELEMENT_TAGS
+    ANS: ansTypes,
+    get ANSType () { return ANSType; },
+    get MigrationStatus () { return MigrationStatus; },
+    get SummarySortBy () { return SummarySortBy; },
+    get SummarySortOrder () { return SummarySortOrder; },
+    TypeUtils: utils
 });
 
-const isTextNode = (node) => {
-    return node instanceof TextNode;
-};
-const isHTMLElement = (node) => {
-    return node instanceof HTMLElement;
-};
-const isCommentNode = (node) => {
-    return node instanceof CommentNode;
-};
-const nodeTagIs = (node, name) => {
-    return isHTMLElement(node) && node.tagName?.toLowerCase() === name.toLowerCase();
-};
-const nodeTagIn = (node, names) => {
-    return isHTMLElement(node) && names.includes(node.tagName?.toLowerCase());
-};
-const isTextCE = (ce) => {
-    return ce?.type === 'text';
-};
-const decodeHTMLEntities = (str) => decode(str);
-const htmlToText = (html, parseOptions) => {
-    if (!html)
-        return '';
-    const doc = parse(html, parseOptions);
-    return decodeHTMLEntities(doc.innerText);
-};
-const getHTMLElementAttribute = (e, key) => {
-    const value = e.getAttribute(key);
-    if (value)
-        return value;
-    return new URLSearchParams(e.rawAttrs.replaceAll(' ', '&')).get(key);
+const reference = (ref) => {
+    return {
+        _id: ref.id,
+        type: 'reference',
+        referent: {
+            ...ref,
+        },
+    };
 };
 
-var html_utils = /*#__PURE__*/Object.freeze({
+var ANS = /*#__PURE__*/Object.freeze({
     __proto__: null,
-    decodeHTMLEntities: decodeHTMLEntities,
-    getHTMLElementAttribute: getHTMLElementAttribute,
-    htmlToText: htmlToText,
-    isCommentNode: isCommentNode,
-    isHTMLElement: isHTMLElement,
-    isTextCE: isTextCE,
-    isTextNode: isTextNode,
-    nodeTagIn: nodeTagIn,
-    nodeTagIs: nodeTagIs
+    reference: reference
 });
 
+const generateArcId = (identifier, orgHostname) => {
+    const namespace = v5(orgHostname, v5.DNS);
+    const buffer = v5(identifier, namespace, Buffer.alloc(16));
+    return encode(buffer, 'RFC4648', { padding: false });
+};
 /**
- * HTMLProcessor is responsible for parsing HTML content into structured content elements.
- * It provides a flexible way to handle different HTML nodes and wrap text content.
- *
- * The processor can be extended with custom handlers for specific node types and
- * wrappers for text content.
+ * Utility class for generating Arc IDs and source IDs
  *
  * @example
  * ```ts
- * // Create and initialize processor
- * const processor = new HTMLProcessor();
- * processor.init();
- *
- * // Parse HTML content
- * const html = '<div><p>Some text</p><img src="image.jpg"></div>';
- * const elements = await processor.parse(html);
+ * const generator = new IdGenerator(['my-org']);
+ * const arcId = generator.getArcId('123'); // Generates a unique for 'my-org' Arc ID
+ * const sourceId = generator.getSourceId('123', ['my-site']); // Generates 'my-site-123'
  * ```
- *
- * The processor comes with built-in handlers for common HTML elements like links,
- * text formatting (i, u, strong), and block elements. Custom handlers can be added
- * using the `handle()` and `wrap()` methods.
  */
-class HTMLProcessor {
-    constructor() {
-        this.parallelProcessing = true;
-        this.handlers = {
-            node: new Map(),
-            wrap: new Map(),
-        };
-    }
-    init() {
-        // wrappers are used to wrap the content of nested text nodes
-        // in a specific way
-        this.wrap('link', (node, text) => {
-            if (nodeTagIn(node, ['a'])) {
-                const attributes = ['href', 'target', 'rel']
-                    .map((attr) => [attr, getHTMLElementAttribute(node, attr)])
-                    .filter(([_, value]) => value)
-                    .map(([key, value]) => `${key}="${value}"`)
-                    .join(' ');
-                return {
-                    ...text,
-                    content: `<a ${attributes}>${text.content}</a>`,
-                };
-            }
-        });
-        this.wrap('i', (node, text) => {
-            if (nodeTagIn(node, ['i'])) {
-                return {
-                    ...text,
-                    content: `<i>${text.content}</i>`,
-                };
-            }
-        });
-        this.wrap('u', (node, text) => {
-            if (nodeTagIn(node, ['u'])) {
-                return {
-                    ...text,
-                    content: `<u>${text.content}</u>`,
-                };
-            }
-        });
-        this.wrap('sup/sub', (node, text) => {
-            if (nodeTagIn(node, ['sup', 'sub'])) {
-                return {
-                    ...text,
-                    content: `<mark class="${node.tagName.toLowerCase()}">${text.content}</mark>`,
-                };
-            }
-        });
-        this.wrap('strong', (node, text) => {
-            if (nodeTagIn(node, ['strong', 'b'])) {
-                return {
-                    ...text,
-                    content: `<b>${text.content}</b>`,
-                };
-            }
-        });
-        this.wrap('center', (node, text) => {
-            if (nodeTagIn(node, ['center'])) {
-                return {
-                    ...text,
-                    alignment: 'center',
-                };
-            }
-        });
-        this.wrap('aligned-paragraph', (node, text) => {
-            if (nodeTagIn(node, ['p'])) {
-                const styleAttribute = getHTMLElementAttribute(node, 'style') || '';
-                if (!styleAttribute)
-                    return text;
-                if (styleAttribute.includes('text-align: right;')) {
-                    return {
-                        ...text,
-                        alignment: 'right',
-                    };
-                }
-                if (styleAttribute.includes('text-align: left;')) {
-                    return {
-                        ...text,
-                        alignment: 'left',
-                    };
-                }
-                if (styleAttribute.includes('text-align: center;')) {
-                    return {
-                        ...text,
-                        alignment: 'center',
-                    };
-                }
-                return text;
-            }
-        });
-        // handlers are used to handle specific nodes
-        // and return a list of content elements
-        this.handle('default', (node) => {
-            const noTag = isHTMLElement(node) && !node.tagName;
-            if (noTag ||
-                nodeTagIn(node, [
-                    'p',
-                    'a',
-                    'b',
-                    'sup',
-                    'sub',
-                    'span',
-                    'strong',
-                    'em',
-                    'i',
-                    'u',
-                    'section',
-                    'main',
-                    'div',
-                    'li',
-                    'center',
-                ])) {
-                return this.handleNested(node);
-            }
-        });
-        this.handle('headers', (node) => {
-            if (nodeTagIn(node, ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'])) {
-                return this.createHeader(node);
-            }
-        });
-        this.handle('text', (node) => {
-            if (isTextNode(node)) {
-                return this.createText(node);
-            }
-        });
-        this.handle('comment', (node) => {
-            if (isCommentNode(node)) {
-                return this.handleComment(node);
-            }
-        });
-        this.handle('list', async (node) => {
-            if (nodeTagIn(node, ['ul', 'ol'])) {
-                const listType = node.tagName === 'UL' ? 'unordered' : 'ordered';
-                return this.createList(node, listType);
-            }
-        });
-        this.handle('table', (node) => {
-            if (nodeTagIs(node, 'table')) {
-                return this.handleTable(node);
-            }
-        });
-        this.handle('iframe', (node) => {
-            if (nodeTagIs(node, 'iframe')) {
-                return this.handleIframe(node);
-            }
-        });
-        this.handle('img', (node) => {
-            if (nodeTagIs(node, 'img')) {
-                return this.handleImage(node);
-            }
-        });
-        this.handle('br', (node) => {
-            if (nodeTagIs(node, 'br')) {
-                return this.handleBreak(node);
-            }
-        });
-    }
-    handle(name, handler) {
-        if (this.handlers.node.has(name)) {
-            this.warn({ name }, `${name} node handler already set`);
-        }
-        this.handlers.node.set(name, handler);
-    }
-    wrap(name, handler) {
-        if (this.handlers.wrap.has(name)) {
-            this.warn({ name }, `${name} wrap handler already set`);
+class IdGenerator {
+    constructor(namespaces) {
+        if (!namespaces.length) {
+            throw new Error('At least 1 namespace is required');
         }
-        this.handlers.wrap.set(name, handler);
+        this.namespace = namespaces.join('-');
     }
-    async parse(html) {
-        const doc = parse(html, { comment: true });
-        doc.removeWhitespace();
-        const elements = await this.process(doc);
-        const filtered = elements?.filter((e) => e.type !== 'divider');
-        return filtered || [];
+    getArcId(id) {
+        return generateArcId(id.toString(), this.namespace);
     }
-    addTextAdditionalProperties(c, parent) {
-        const additionalProperties = c.additional_properties || {};
-        const parentNodeIsBlockElement = this.isBlockElement(parent);
-        c.additional_properties = {
-            ...c.additional_properties,
-            isBlockElement: additionalProperties.isBlockElement || parentNodeIsBlockElement,
-        };
-        return c;
+    getSourceId(id, prefixes = []) {
+        return [...prefixes, id].join('-');
     }
-    /**
-     * Wraps text content elements with additional properties and handlers.
-     * This method iterates through an array of content elements and applies
-     * wrappers to text elements.
-     *
-     * @param node - The HTML node containing the text elements
-     **/
-    wrapChildrenTextNodes(node, elements) {
-        const wrapped = [];
-        const wrappers = [...this.handlers.wrap.values()];
-        for (const c of elements) {
-            if (!isTextCE(c)) {
-                wrapped.push(c);
-                continue;
-            }
-            this.addTextAdditionalProperties(c, node);
-            const handled = wrappers.map((wrapper) => wrapper(node, c)).find(Boolean);
-            wrapped.push(handled || c);
+}
+
+var Id = /*#__PURE__*/Object.freeze({
+    __proto__: null,
+    IdGenerator: IdGenerator,
+    generateArcId: generateArcId
+});
+
+const buildTree = (items) => {
+    const tree = [
+        {
+            id: '/',
+            children: [],
+            meta: new Proxy({}, {
+                get: () => {
+                    throw new Error('Root node meta is not accessible');
+                },
+            }),
+            parent: null,
+        },
+    ];
+    // Track nodes at each level to maintain parent-child relationships
+    // stores last node at each level
+    const currLevelNodes = {
+        0: tree[0],
+    };
+    for (const item of items) {
+        const node = {
+            id: item.id,
+            parent: null,
+            children: [],
+            meta: item,
+        };
+        // Determine the level of this node
+        const levelKey = Object.keys(item).find((key) => key.startsWith('N') && item[key]);
+        const level = Number(levelKey?.replace('N', '')) || 0;
+        if (!level) {
+            throw new Error(`Invalid level for section ${item.id}`);
         }
-        return wrapped;
+        // This is a child node - attach to its parent
+        const parentLevel = level - 1;
+        const parentNode = currLevelNodes[parentLevel];
+        if (parentNode) {
+            node.parent = parentNode;
+            parentNode.children.push(node);
+        }
+        else {
+            throw new Error(`Parent node not found for section ${item.id}`);
+        }
+        // Set this as the current node for its level
+        currLevelNodes[level] = node;
     }
-    /**
-     * Handles nested nodes by processing their children and merging text elements.
-     * This method recursively processes the children of a given HTML node and
-     * returns a list of content elements.
-     *
-     * @param node - The HTML node to process
-     **/
-    async handleNested(node) {
-        const children = await this.processChildNodes(node);
-        const filtered = children.filter(Boolean).flat();
-        const merged = this.mergeParagraphs(filtered);
-        const wrapped = this.wrapChildrenTextNodes(node, merged);
-        return wrapped;
+    // return root nodes children
+    return tree[0].children;
+};
+const flattenTree = (tree) => {
+    const flatten = [];
+    const traverse = (node) => {
+        flatten.push(node);
+        for (const child of node.children) {
+            traverse(child);
+        }
+    };
+    // traverse all root nodes and their children
+    for (const node of tree) {
+        traverse(node);
     }
-    async processChildNodes(node) {
-        if (this.parallelProcessing) {
-            return await Promise.all(node.childNodes.map((child) => this.process(child)));
+    return flatten;
+};
+const buildAndFlattenTree = (items) => flattenTree(buildTree(items));
+const groupByWebsites = (sections) => {
+    return sections.reduce((acc, section) => {
+        const website = section._website;
+        if (!acc[website])
+            acc[website] = [];
+        acc[website].push(section);
+        return acc;
+    }, {});
+};
+const references = (sections) => {
+    return sections.map((s) => reference({
+        id: s._id,
+        website: s._website,
+        type: 'section',
+    }));
+};
+const isReference = (section) => {
+    return section?.type === 'reference' && section?.referent?.type === 'section';
+};
+const removeDuplicates = (sections) => {
+    const map = new Map();
+    sections.forEach((s) => {
+        if (isReference(s)) {
+            map.set(`${s.referent.id}${s.referent.website}`, s);
         }
-        const children = [];
-        for (const child of node.childNodes) {
-            children.push(await this.process(child));
+        else {
+            map.set(`${s._id}${s._website}`, s);
         }
-        return children;
+    });
+    return [...map.values()];
+};
+class SectionsRepository {
+    constructor(arc) {
+        this.arc = arc;
+        this.sectionsByWebsite = {};
+        this.websitesAreLoaded = false;
     }
-    /**
-     * Processes a single HTML node and converts it into content elements.
-     * This method iterates through registered node handlers and attempts to process the node.
-     * If a handler successfully processes the node, it returns an array of content elements.
-     *
-     * @param node - The HTML node to process
-     * @returns Promise resolving to an array of content elements, or undefined if node cannot be processed
-     */
-    async process(node) {
-        let isKnownNode = false;
-        const elements = [];
-        for (const [name, handler] of this.handlers.node.entries()) {
-            try {
-                const result = await handler(node);
-                if (result) {
-                    // if handler returns an array of elements, it means that the node was handled properly, even if there is no elements inside
-                    isKnownNode = true;
-                    elements.push(...result);
-                    break;
-                }
-            }
-            catch (error) {
-                this.warn({ node: node.toString(), error: error.toString(), name }, 'HandlerError');
-            }
-        }
-        if (isKnownNode)
-            return elements;
-        this.warn({ node: node.toString() }, 'UnknownNodeError');
+    async put(ans) {
+        await this.arc.Site.putSection(ans);
+        const created = await this.arc.Site.getSection(ans._id, ans.website);
+        this.save(created);
     }
-    /**
-     * Merges adjacent text content elements into a single paragraph.
-     * This method iterates through an array of content elements and combines
-     * adjacent text elements into a single paragraph.
-     *
-     * @param items - The array of content elements to merge
-     **/
-    mergeParagraphs(items) {
-        const merged = [];
-        let toMerge = [];
-        const merge = () => {
-            if (!toMerge.length)
-                return;
-            const paragraph = toMerge.reduce((acc, p) => {
-                return {
-                    ...p,
-                    content: acc.content + p.content,
-                };
-            }, { type: 'text', content: '' });
-            merged.push(paragraph);
-            toMerge = [];
-        };
-        for (let i = 0; i < items.length; i++) {
-            const item = items[i];
-            const isBlockElement = item.additional_properties?.isBlockElement;
-            if (isTextCE(item) && !isBlockElement) {
-                toMerge.push(item);
+    async loadWebsite(website) {
+        const sections = [];
+        let next = true;
+        let offset = 0;
+        while (next) {
+            const migrated = await this.arc.Site.getSections({ website, offset }).catch((_) => {
+                return { q_results: [] };
+            });
+            if (migrated.q_results.length) {
+                sections.push(...migrated.q_results);
+                offset += migrated.q_results.length;
             }
             else {
-                merge();
-                merged.push(item);
+                next = false;
             }
         }
-        merge();
-        return merged;
-    }
-    handleComment(_) {
-        return [];
-    }
-    async handleTable(node) {
-        return [ContentElement.raw_html(node.toString())];
-    }
-    async handleIframe(node) {
-        return [ContentElement.raw_html(node.toString())];
-    }
-    async handleImage(node) {
-        return [ContentElement.raw_html(node.toString())];
-    }
-    async handleBreak(_) {
-        return [ContentElement.divider()];
-    }
-    async createQuote(node) {
-        const items = await this.handleNested(node);
-        return [ContentElement.quote(items)];
+        return sections;
     }
-    async createText(node) {
-        const text = ContentElement.text(node.text);
-        return [text];
+    async loadWebsites(websites) {
+        for (const website of websites) {
+            this.sectionsByWebsite[website] = await this.loadWebsite(website);
+        }
+        this.websitesAreLoaded = true;
     }
-    filterListItems(items) {
-        return items.filter((i) => ['text', 'list'].includes(i.type));
+    save(section) {
+        const website = section._website;
+        assert.ok(website, 'Section must have a website');
+        this.sectionsByWebsite[website] = this.sectionsByWebsite[website] || [];
+        if (!this.sectionsByWebsite[website].find((s) => s._id === section._id)) {
+            this.sectionsByWebsite[website].push(section);
+        }
     }
-    async createList(node, type) {
-        const items = await this.handleNested(node);
-        return [ContentElement.list(type, this.filterListItems(items))];
+    getById(id, website) {
+        this.ensureWebsitesLoaded();
+        const section = this.sectionsByWebsite[website]?.find((s) => s._id === id);
+        return section;
     }
-    async createHeader(node) {
-        const level = +node.tagName.split('H')[1] || 3;
-        return [ContentElement.header(node.innerText, level)];
+    getByWebsite(website) {
+        this.ensureWebsitesLoaded();
+        return this.sectionsByWebsite[website];
     }
-    isBlockElement(node) {
-        if (!isHTMLElement(node))
-            return false;
-        const defaultBlockElements = new Set(BLOCK_ELEMENT_TAGS);
-        return defaultBlockElements.has(node.tagName);
+    getParentSections(section) {
+        this.ensureWebsitesLoaded();
+        const parents = [];
+        let current = section;
+        while (current.parent?.default && current.parent.default !== '/') {
+            const parent = this.getById(current.parent.default, section._website);
+            if (!parent)
+                break;
+            parents.push(parent);
+            current = parent;
+        }
+        return parents;
     }
-    warn(metadata, message) {
-        console.warn(metadata, message);
+    ensureWebsitesLoaded() {
+        assert.ok(this.websitesAreLoaded, 'call .loadWebsites() first');
     }
 }
 
-var index$1 = /*#__PURE__*/Object.freeze({
+var Section = /*#__PURE__*/Object.freeze({
     __proto__: null,
-    Constants: html_constants,
-    HTMLProcessor: HTMLProcessor,
-    Utils: html_utils
+    SectionsRepository: SectionsRepository,
+    buildAndFlattenTree: buildAndFlattenTree,
+    buildTree: buildTree,
+    flattenTree: flattenTree,
+    groupByWebsites: groupByWebsites,
+    isReference: isReference,
+    references: references,
+    removeDuplicates: removeDuplicates
 });
 
-var index = /*#__PURE__*/Object.freeze({
-    __proto__: null,
-    ContentElement: ContentElement,
-    HTML: index$1
-});
+const ArcUtils = {
+    Id,
+    ANS,
+    ContentElements,
+    Section,
+};
 
-export { index$2 as AnsMapper, ArcAPI, ArcError, index$3 as ArcTypes, ArcUtils, index as ContentElements, WsClient, ArcAPI as default };
+export { index$1 as AnsMapper, ArcAPI, ArcError, index as ArcTypes, ArcUtils, index$2 as ContentElements, WsClient, ArcAPI as default };
 //# sourceMappingURL=index.js.map