@tricoteuses/tisseuse 0.13.9 → 0.13.11

package/README.md

@@ -96,6 +96,17 @@ npm run index:postgres:assemblee -- --all
 
 La première commande consomme la file des documents à acquérir, simplifier et enrichir. La seconde indexe dans `tisseuse.assemblee_text_search` les articles et les exposés des motifs comme résultats distincts. Les déclencheurs sur `assemblee.documents`, le hash du contenu et les files PostgreSQL permettent les mises à jour ciblées ; les options `--all` demandent une réconciliation complète.
 
+Pour acquérir et indexer immédiatement quelques documents indépendamment de leur position dans les files durables :
+
+```sh
+npm run sync:postgres:assemblee-content -- \
+  --document-uids PRJLANR5L17BTC2765,PRJLANR5L17BTC3067
+npm run index:postgres:assemblee -- \
+  --document-uids PRJLANR5L17BTC2765,PRJLANR5L17BTC3067
+```
+
+Le mode ciblé refuse les UID absents de `assemblee.documents` et ne peut pas être combiné avec `--all`.
+
 Exploratrice et Moulineuse ne téléchargent jamais un document pendant une lecture. Lorsqu’un document source existe mais que son contenu n’est pas encore disponible, son état est conservé en base et l’interface doit l’indiquer au lieu de servir une copie locale périmée.
 
 ### Première itération de recherche plein texte Sénat

package/dist/server.js

@@ -247,7 +247,7 @@ function ge(e, t) {
 }
 function P(e) {
 	let t = [], n = /* @__PURE__ */ new Set(), r = /* @__PURE__ */ new Set();
-	return e("a[id], span[id], h1[id], h2[id], h3[id], h4[id], h5[id], h6[id]").each((i, a) => {
+	e("a[id], span[id], h1[id], h2[id], h3[id], h4[id], h5[id], h6[id]").each((i, a) => {
 		let o = M(E(e(a).text()));
 		if (!o) return;
 		let s = e(a).attr("id");
@@ -259,7 +259,20 @@ function P(e) {
 			element: a,
 			container: c
 		}));
-	}), t;
+	}), e("h1:not([id]), h2:not([id]), h3:not([id]), h4:not([id]), h5:not([id]), h6:not([id])").each((i, a) => {
+		if (n.has(a)) return;
+		let o = M(E(e(a).text()));
+		if (!o) return;
+		let s = `_TocArt${(o.num ?? o.label.replace(/^Article\s+/iu, "")).replace(/\s+/gu, "")}`;
+		r.has(s) || (r.add(s), n.add(a), t.push({
+			id: s,
+			...o,
+			element: a,
+			container: a
+		}));
+	});
+	let i = e("body *");
+	return t.sort((e, t) => i.index(e.container) - i.index(t.container)), t;
 }
 function F(e) {
 	let t = e.next;
@@ -269,7 +282,7 @@ function F(e) {
 	}
 	return null;
 }
-function _e(e, t, n) {
+function I(e, t, n) {
 	let r = [t.container], i = n?.container;
 	for (let e = F(t.container); e !== null && e !== i; e = F(e)) r.push(e);
 	if (r.length > 1) return r;
@@ -279,7 +292,7 @@ function _e(e, t, n) {
 		return n >= a && (o === -1 || n < o);
 	});
 }
-function ve(e, t) {
+function _e(e, t) {
 	let n = [], r = /* @__PURE__ */ new Set();
 	for (let i of t) {
 		let t = "p,li,table", a = e(i).is(t) ? [i] : e(i).find(t).get();
@@ -287,9 +300,9 @@ function ve(e, t) {
 	}
 	return n;
 }
-function I(e, t, n) {
+function L(e, t, n) {
 	let r = [], i = !1;
-	for (let a of ve(e, n)) {
+	for (let a of _e(e, n)) {
 		let n = E(e(a).text());
 		if (!n) continue;
 		T.test(n) && (i = !0);
@@ -305,52 +318,64 @@ function I(e, t, n) {
 	}
 	return r;
 }
-function L(e, t) {
+function R(e, t) {
 	if (t.length === 0) return e;
 	let n = `<div style="display: none;">\n${t.map((e) => `<p class="assnatTOC6"><a href="#${`_TocArt${(e.num ?? e.label.replace(/^Article\s+/iu, "")).replace(/\s+/gu, "")}`}"><span class="assnatHyperlink" style="font-weight:bold; text-decoration:none; color:#000000">${e.label.toUpperCase()}</span></a></p>`).join("\n")}\n</div>`;
 	return e.includes("class=\"assnatSection3\"") ? e.replace(/(<div class="assnatSection3">)/u, `${n}$1`) : e.includes("id=\"formCorrection:panel:1:table\"") ? e.replace(/(<div id="formCorrection:panel:1:table">)/u, `${n}$1`) : `${n}\n${e}`;
 }
-function ye(t) {
+function ve(t) {
 	let n = e(t);
 	for (let e of P(n)) {
 		let t = `_TocArt${(e.num ?? e.label.replace(/^Article\s+/iu, "")).replace(/\s+/gu, "")}`, r = n(e.element), i = r.attr("id");
-		i?.startsWith("_TocArt") || (i?.startsWith("_Toc") && !be(n, i, e.element) && r.before(n("<span></span>").attr("id", i)), r.attr("id", t));
+		i?.startsWith("_TocArt") || (i?.startsWith("_Toc") && !z(n, i, e.element) && r.before(n("<span></span>").attr("id", i)), r.attr("id", t));
+	}
+	let r = P(n);
+	for (let [e, t] of r.entries()) {
+		let i = I(n, t, r[e + 1]).flatMap((e) => n(e).is("span.alinea[data-alinea]") ? [e] : n(e).find("span.alinea[data-alinea]").get());
+		for (let e of i) {
+			let r = n(e).attr("data-alinea")?.trim();
+			if (!r) continue;
+			let i = `${t.id}-alinea-${r.replace(/\s+/gu, "")}`;
+			if (z(n, i, e)) continue;
+			let a = n(e).attr("id");
+			a && a !== i ? n(e).before(n("<span></span>").attr("id", i)) : n(e).attr("id", i);
+		}
 	}
 	return n.html();
 }
-function be(e, t, n) {
+function z(e, t, n) {
 	for (let r of e("[id]").get()) if (r !== n && e(r).attr("id") === t) return !0;
 	return !1;
 }
-function xe(e, t) {
+function ye(e, t) {
 	let n = ue("sourceText" in t ? t.sourceText : "");
 	if (n) return e.filter((e) => e.kind === "directive_candidate").find((e) => {
 		let t = ue(e.text);
 		return t.length > 0 && (n.includes(t) || t.includes(n));
 	});
 }
-function Se(e, t) {
+function be(e, t) {
 	if (t) return t.blockIds.map((t) => e.get(t)).find((e) => e?.kind === "directive_candidate");
 }
-function R(e) {
+function xe(e) {
 	return e?.links.filter((e) => !e.insideFrenchQuotes && (e.type === "article" || e.type === "division")).map((e) => e.id) ?? [];
 }
-var Ce = [
+var Se = [
 	/^([IVXLCDM]+)\s*[.)．]?\s*[–—-]/iu,
 	/^(\d+°)\s+/u,
 	/^([a-z])\)\s+/iu,
 	/^([ivxlcdm]+)\)\s+/iu
 ];
-function we(e) {
+function Ce(e) {
 	let t = E(e);
 	if (!t) return;
-	let n = Ce.findIndex((e) => e.test(t));
+	let n = Se.findIndex((e) => e.test(t));
 	return n === -1 ? t.endsWith(":") ? 0 : void 0 : n + 1;
 }
-function Te(e) {
+function we(e) {
 	let t = /* @__PURE__ */ new Map(), n = [];
 	for (let r of e) {
-		let e = we(r.text);
+		let e = Ce(r.text);
 		if (e === void 0) {
 			t.set(r.id, [...n, r]);
 			continue;
@@ -359,14 +384,14 @@ function Te(e) {
 	}
 	return t;
 }
-function Ee(e, t) {
+function Te(e, t) {
 	let n = [], r = /* @__PURE__ */ new Set();
 	for (let i of [...e ?? [], ...t]) i !== void 0 && (r.has(i.id) || (r.add(i.id), n.push(i)));
 	return n;
 }
-function De(e, t, n) {
+function Ee(e, t, n) {
 	if (!t) return;
-	let r = Ee(e, t.blockIds.map((e) => n.get(e)).filter((e) => e !== void 0));
+	let r = Te(e, t.blockIds.map((e) => n.get(e)).filter((e) => e !== void 0));
 	if (r.length === 0) return;
 	let i = r.map((e) => e.id);
 	if (i.length === t.blockIds.length && i.every((e, n) => e === t.blockIds[n])) return;
@@ -381,7 +406,7 @@ function De(e, t, n) {
 		}
 	};
 }
-function Oe(e) {
+function De(e) {
 	let t = [], n = 0;
 	for (let r of e.blocks) {
 		let i = e.directiveText.indexOf(r.text, n);
@@ -395,7 +420,7 @@ function Oe(e) {
 	}
 	return t;
 }
-function ke(e, t, n) {
+function Oe(e, t, n) {
 	let { start: r, stop: i } = n.sourcePosition, a = t.filter((e) => e.stop >= r && e.start <= i);
 	if (a.length === 0) return;
 	let o = a[0], s = a[a.length - 1], c = o.start, l = s.stop;
@@ -409,7 +434,7 @@ function ke(e, t, n) {
 		}
 	};
 }
-function Ae(e, t, n) {
+function ke(e, t, n) {
 	if (!t) return;
 	let r = e.directiveText.indexOf(t.text);
 	if (r === -1) return;
@@ -422,14 +447,14 @@ function Ae(e, t, n) {
 		stop: a
 	};
 }
-function je(e) {
-	let t = Oe(e), n = new Map(e.blocks.map((e) => [e.id, e])), r = Te(e.blocks), a = /* @__PURE__ */ new Map(), o = [];
+function Ae(e) {
+	let t = De(e), n = new Map(e.blocks.map((e) => [e.id, e])), r = we(e.blocks), a = /* @__PURE__ */ new Map(), o = [];
 	for (let t of e.blocks) {
-		let e = R(t);
+		let e = xe(t);
 		e.length > 0 && (o = e), a.set(t.id, o);
 	}
 	return i(e.directiveText).map((i) => {
-		let o = ke(e, t, i), s = Se(n, o) ?? xe(e.blocks, i), c = R(s), l = c.length > 0 ? c : s?.id === void 0 ? [] : a.get(s.id) ?? [], u = Ae(e, s, i), d = De(s?.id === void 0 ? void 0 : r.get(s.id), o, n);
+		let o = Oe(e, t, i), s = be(n, o) ?? ye(e.blocks, i), c = xe(s), l = c.length > 0 ? c : s?.id === void 0 ? [] : a.get(s.id) ?? [], u = ke(e, s, i), d = Ee(s?.id === void 0 ? void 0 : r.get(s.id), o, n);
 		return {
 			articleId: e.id,
 			blockId: s?.id,
@@ -449,7 +474,7 @@ function je(e) {
 		};
 	});
 }
-function Me(e) {
+function je(e) {
 	let t = /* @__PURE__ */ new Map(), n = [], r = /* @__PURE__ */ new Set();
 	for (let i of e) {
 		let e = re(i.directive);
@@ -479,7 +504,7 @@ function Me(e) {
 		definitions: n
 	};
 }
-function Ne(e, t, n) {
+function Me(e, t, n) {
 	let r = new Map(t.map((e) => [O(e.num), e]));
 	return e.map((e) => {
 		let t = n.get(e), i = t?.[0], a = [...k(e.directive.reference)].map((e) => r.get(O(e))).filter((e) => e !== void 0 && e.definitionId !== i?.definitionId)[0], o = a === void 0 ? void 0 : {
@@ -498,7 +523,7 @@ function Ne(e, t, n) {
 		};
 	});
 }
-function z(e, t, n, i, a, o, s = !0) {
+function Ne(e, t, n, i, a, o, s = !0) {
 	let c = a.map((e) => e.directiveText).filter(Boolean).join("\n"), l = {
 		sourceFormat: e,
 		documentDate: i,
@@ -511,7 +536,7 @@ function z(e, t, n, i, a, o, s = !0) {
 		...l,
 		directives: []
 	};
-	let u = le(a.map((e) => e.text).join("\n")), d = a.flatMap(je), { createdArticlesByDirective: f, definitions: p } = Me(d), m = Ne(d, p, f);
+	let u = le(a.map((e) => e.text).join("\n")), d = a.flatMap(Ae), { createdArticlesByDirective: f, definitions: p } = je(d), m = Me(d, p, f);
 	return {
 		...l,
 		...u ? { globalApplicabilityDate: u } : {},
@@ -532,8 +557,8 @@ function Pe(e) {
 	})) };
 }
 async function Fe(t, n) {
-	let r = new u("parseAssembleeBillRaw"), i = n.linkedHtml === void 0 ? await r.time("linkAssembleeBillRaw", () => Ie(t, n)) : void 0, a = r.timeSync("applyArticleAnchorIds", () => ye(n.linkedHtml ?? i.linkedHtml), { inputSize: t.length }), o = r.timeSync("cheerioLoadLinkedHtml", () => e(a)), s = r.timeSync("extractArticleHeadings", () => P(o)), c = r.timeSync("extractArticlesAndBlocks", () => s.map((e, t) => {
-		let n = _e(o, e, s[t + 1]), r = I(o, e.id, n), i = r.filter((e) => e.kind !== "noise").map((e) => e.text).join("\n");
+	let r = new u("parseAssembleeBillRaw"), i = n.linkedHtml === void 0 ? await r.time("linkAssembleeBillRaw", () => Ie(t, n)) : void 0, a = r.timeSync("applyArticleAnchorIds", () => ve(n.linkedHtml ?? i.linkedHtml), { inputSize: t.length }), o = r.timeSync("cheerioLoadLinkedHtml", () => e(a)), s = r.timeSync("extractArticleHeadings", () => P(o)), c = r.timeSync("extractArticlesAndBlocks", () => s.map((e, t) => {
+		let n = I(o, e, s[t + 1]), r = L(o, e.id, n), i = r.filter((e) => e.kind !== "noise").map((e) => e.text).join("\n");
 		return {
 			id: e.id,
 			label: e.label,
@@ -544,7 +569,7 @@ async function Fe(t, n) {
 			directiveText: i,
 			blocks: r
 		};
-	}), { count: s.length }), l = r.timeSync("injectHiddenArticleSummary", () => L(o.html(), c)), d = r.timeSync("buildAssembleeTableOfContents", () => Pe(c), { count: c.length }), f = r.timeSync("buildParsedDocument", () => z("assemblee-html", t, l, n.date, c, d, n.includeDirectiveAnalysis));
+	}), { count: s.length }), l = r.timeSync("injectHiddenArticleSummary", () => R(o.html(), c)), d = r.timeSync("buildAssembleeTableOfContents", () => Pe(c), { count: c.length }), f = r.timeSync("buildParsedDocument", () => Ne("assemblee-html", t, l, n.date, c, d, n.includeDirectiveAnalysis));
 	return r.log({
 		articleCount: f.articles.length,
 		blockCount: f.articles.reduce((e, t) => e + t.blocks.length, 0),
@@ -966,7 +991,7 @@ function gt(t) {
 function _t(e) {
 	let t = [];
 	return e("article.akn-article").each((n, r) => {
-		let i = e(r).children("p.TCNumArticle").first().get(0), a = E(i ? e(i).text() : ""), o = M(a) ?? { label: a || `Article ${n + 1}` }, s = e(r).attr("id") ?? `akn-article-${String(n + 1).padStart(3, "0")}`, c = [r], l = I(e, s, c), u = l.filter((e) => e.kind !== "noise").map((e) => e.text).join("\n");
+		let i = e(r).children("p.TCNumArticle").first().get(0), a = E(i ? e(i).text() : ""), o = M(a) ?? { label: a || `Article ${n + 1}` }, s = e(r).attr("id") ?? `akn-article-${String(n + 1).padStart(3, "0")}`, c = [r], l = L(e, s, c), u = l.filter((e) => e.kind !== "noise").map((e) => e.text).join("\n");
 		t.push({
 			id: s,
 			label: o.label,
@@ -995,7 +1020,7 @@ async function yt(t, n) {
 	} })), a = n.date ?? pt(i), o = r.timeSync("renderAknHtml", () => mt(i), { inputSize: t.length }), s = n.linkedHtml ?? await r.time("addLinksToRenderedAknHtml", () => N(o, {
 		...n,
 		date: a
-	}, a)), c = r.timeSync("extractRenderedAknArticles", () => gt(o)), l = r.timeSync("injectHiddenArticleSummary", () => L(s, c)), { articles: d, htmlWithLinks: f } = r.timeSync("extractLinkedAknArticlesAndTaggedHtml", () => vt(l)), p = d.length > 0 ? d : c, m = r.timeSync("extractAknTableOfContents", () => ft(i)), h = r.timeSync("buildParsedDocument", () => z("akoma-ntoso", o, d.length > 0 ? f : l, a, p, m));
+	}, a)), c = r.timeSync("extractRenderedAknArticles", () => gt(o)), l = r.timeSync("injectHiddenArticleSummary", () => R(s, c)), { articles: d, htmlWithLinks: f } = r.timeSync("extractLinkedAknArticlesAndTaggedHtml", () => vt(l)), p = d.length > 0 ? d : c, m = r.timeSync("extractAknTableOfContents", () => ft(i)), h = r.timeSync("buildParsedDocument", () => Ne("akoma-ntoso", o, d.length > 0 ? f : l, a, p, m));
 	return r.log({
 		articleCount: h.articles.length,
 		blockCount: h.articles.reduce((e, t) => e + t.blocks.length, 0),

package/dist/src/lib/server/atomic_files.d.ts

@@ -0,0 +1,8 @@
+export declare function writeJsonAtomic(filePath: string, value: unknown, options?: {
+    encoding?: BufferEncoding;
+    spaces?: number;
+    stagingDirectory?: string;
+}): Promise<void>;
+export declare function replaceDirectoryAtomically(targetDirectory: string, build: (stagingDirectory: string) => Promise<void>, { stagingRoot, }?: {
+    stagingRoot?: string;
+}): Promise<void>;

package/dist/src/lib/server/atomic_files.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/lib/server/document_git_checkpoint.d.ts

@@ -0,0 +1,6 @@
+export declare function commitAndPushDocumentCheckpoint({ branch, message, remote, repository, }: {
+    branch?: string;
+    message: string;
+    remote?: string;
+    repository: string;
+}): number;

package/dist/src/lib/server/document_git_checkpoint.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/lib/server/enriched_document_checkpoint.d.ts

@@ -0,0 +1,3 @@
+export declare const enrichedDocumentCompletionMarkerName = ".tricoteuses-complete";
+export declare const legacyEnrichedDocumentRequiredOutputNames: string[];
+export declare function enrichedDocumentIsComplete(directory: string): Promise<boolean>;

package/dist/src/lib/server/enriched_document_checkpoint.test.d.ts

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@tricoteuses/tisseuse",
   "description": "Find links in/to French legislative documents",
-  "version": "0.13.9",
+  "version": "0.13.11",
   "keywords": [
     "Assemblée nationale",
     "France",