@nanocollective/get-md 1.3.1 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/converters/llm-converter.d.ts.map +1 -1
- package/dist/converters/llm-converter.js +3 -2
- package/dist/converters/llm-converter.js.map +1 -1
- package/dist/converters/llm-manager.d.ts.map +1 -1
- package/dist/converters/llm-manager.js +2 -1
- package/dist/converters/llm-manager.js.map +1 -1
- package/dist/converters/load-llama.d.ts +2 -0
- package/dist/converters/load-llama.d.ts.map +1 -0
- package/dist/converters/load-llama.js +25 -0
- package/dist/converters/load-llama.js.map +1 -0
- package/package.json +11 -3
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm-converter.d.ts","sourceRoot":"","sources":["../../src/converters/llm-converter.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"llm-converter.d.ts","sourceRoot":"","sources":["../../src/converters/llm-converter.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAOpD;;;;;GAKG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,KAAK,CAA2B;IACxC,OAAO,CAAC,OAAO,CAA6B;IAC5C,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,aAAa,CAAC,CAAmB;IACzC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,SAAS,CAAS;gBAEd,OAAO,EAAE;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,CAAC,EAAE,gBAAgB,CAAC;QAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB;IAOD;;OAEG;YACW,IAAI;IAMlB;;OAEG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IA6ChC;;OAEG;IACH,QAAQ,IAAI,OAAO;IAInB;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAe7B;;OAEG;IACH,OAAO,CAAC,aAAa;IAerB;;;;;OAKG;IACG,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;CAmE7C;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,gBAAgB,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,YAAY,CAEf"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
// src/converters/llm-converter.ts
|
|
2
|
+
import { loadNodeLlamaCpp } from "./load-llama.js";
|
|
2
3
|
// Default conversion parameters
|
|
3
4
|
const DEFAULT_TEMPERATURE = 0;
|
|
4
5
|
const DEFAULT_MAX_TOKENS = 4096;
|
|
@@ -44,7 +45,7 @@ export class LLMConverter {
|
|
|
44
45
|
await this.emit({
|
|
45
46
|
type: "llama-init-start",
|
|
46
47
|
});
|
|
47
|
-
const { getLlama } = await
|
|
48
|
+
const { getLlama } = await loadNodeLlamaCpp();
|
|
48
49
|
this.llama = await getLlama();
|
|
49
50
|
await this.emit({
|
|
50
51
|
type: "llama-init-complete",
|
|
@@ -125,7 +126,7 @@ export class LLMConverter {
|
|
|
125
126
|
});
|
|
126
127
|
try {
|
|
127
128
|
// Create chat session for proper Qwen2.5 chat template handling
|
|
128
|
-
const { LlamaChatSession } = await
|
|
129
|
+
const { LlamaChatSession } = await loadNodeLlamaCpp();
|
|
129
130
|
const session = new LlamaChatSession({
|
|
130
131
|
contextSequence: this.context.getSequence(),
|
|
131
132
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm-converter.js","sourceRoot":"","sources":["../../src/converters/llm-converter.ts"],"names":[],"mappings":"AAAA,kCAAkC;
|
|
1
|
+
{"version":3,"file":"llm-converter.js","sourceRoot":"","sources":["../../src/converters/llm-converter.ts"],"names":[],"mappings":"AAAA,kCAAkC;AASlC,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAEnD,gCAAgC;AAChC,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAC9B,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAEhC;;;;;GAKG;AACH,MAAM,OAAO,YAAY;IACf,KAAK,GAAiB,IAAI,CAAC;IAC3B,KAAK,GAAsB,IAAI,CAAC;IAChC,OAAO,GAAwB,IAAI,CAAC;IACpC,SAAS,CAAS;IAClB,aAAa,CAAoB;IACjC,WAAW,CAAS;IACpB,SAAS,CAAS;IAE1B,YAAY,OAKX;QACC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,OAAO,CAAC;QACrC,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,mBAAmB,CAAC;QAC9D,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAC3D,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,IAAI,CAAC,KAAsC;QACvD,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS;QACb,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,MAAM,IAAI,CAAC,IAAI,CAAC;YACd,IAAI,EAAE,eAAe;YACrB,SAAS,EAAE,aAAa;SACzB,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,8EAA8E;YAC9E,MAAM,IAAI,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,kBAAkB;aACzB,CAAC,CAAC;YACH,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,gBAAgB,EAAE,CAAC;YAC9C,IAAI,CAAC,KAAK,GAAG,MAAM,QAAQ,EAAE,CAAC;YAC9B,MAAM,IAAI,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,qBAAqB;aAC5B,CAAC,CAAC;YAEH,6BAA6B;YAC7B,MAAM,IAAI,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,oBAAoB;gBAC1B,IAAI,EAAE,IAAI,CAAC,SAAS;aACrB,CAAC,CAAC;YACH,IAAI,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;gBACtC,SAAS,EAAE,IAAI,CAAC,SAAS;aAC1B,CAAC,CAAC;YAEH,8DAA8D;YAC9D,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC;gBAC5C,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,wCAAwC;aACtF,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAExC,MAAM,IAAI,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,cAAc;gBACpB,QAAQ;aACT,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YACtE,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,OAAO,IAAI,CAAC,KAAK,KAAK,IAAI,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC;IACtD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM;QACV,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC;QACD,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YAC3B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QACpB,CAAC;QACD,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YAC3B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QACpB,CAAC;IACH,CAAC;IAED;;OAEG;IACK,aAAa,CAAC,MAAc;QAClC,IAAI,OAAO,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;QAE5B,gDAAgD;QAChD,0CAA0C;QAC1C,MAAM,cAAc,GAAG,OAAO,CAAC,KAAK,CAClC,8CAA8C,CAC/C,CAAC;QACF,IAAI,cAAc,EAAE,CAAC;YACnB,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;QAC9B,CAAC;QAED,OAAO,OAAO,CAAC,IAAI,EAAE,CAAC;IACxB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,OAAO,CAAC,IAAY;QACxB,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC/D,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,MAAM,IAAI,CAAC,IAAI,CAAC;YACd,IAAI,EAAE,kBAAkB;YACxB,SAAS,EAAE,IAAI,CAAC,MAAM;SACvB,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,gEAAgE;YAChE,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,gBAAgB,EAAE,CAAC;YACtD,MAAM,OAAO,GAAG,IAAI,gBAAgB,CAAC;gBACnC,eAAe,EAAE,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE;aAC5C,CAAC,CAAC;YAEH,+CAA+C;YAC/C,kEAAkE;YAClE,MAAM,MAAM,GAAG;;EAEnB,IAAI;OACC,CAAC;YAEF,IAAI,eAAe,GAAG,CAAC,CAAC;YAExB,mCAAmC;YACnC,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE;gBAC7C,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,WAAW,EAAE,GAAG,EAAE;oBAChB,eAAe,EAAE,CAAC;oBAClB,gDAAgD;oBAChD,IAAI,eAAe,GAAG,GAAG,KAAK,CAAC,EAAE,CAAC;wBAChC,KAAK,IAAI,CAAC,IAAI,CAAC;4BACb,IAAI,EAAE,qBAAqB;4BAC3B,eAAe;yBAChB,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;aACF,CAAC,CAAC;YAEH,sEAAsE;YACtE,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;YAE7C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YAExC,MAAM,IAAI,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,qBAAqB;gBAC3B,UAAU,EAAE,MAAM,CAAC,MAAM;gBACzB,QAAQ;aACT,CAAC,CAAC;YAEH,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAEtE,MAAM,IAAI,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,kBAAkB;gBACxB,KAAK,EAAE,GAAG;aACX,CAAC,CAAC;YAEH,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;CACF;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAKlC;IACC,OAAO,IAAI,YAAY,CAAC,OAAO,CAAC,CAAC;AACnC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm-manager.d.ts","sourceRoot":"","sources":["../../src/converters/llm-manager.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EACV,kBAAkB,EAClB,gBAAgB,EAChB,YAAY,EACZ,cAAc,EAEf,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"llm-manager.d.ts","sourceRoot":"","sources":["../../src/converters/llm-manager.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EACV,kBAAkB,EAClB,gBAAgB,EAChB,YAAY,EACZ,cAAc,EAEf,MAAM,aAAa,CAAC;AAwErB;;GAEG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,aAAa,CAAC,CAAmB;gBAE7B,OAAO,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,gBAAgB,CAAA;KAAE;IAKxE;;OAEG;YACW,IAAI;IAMlB;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,cAAc,CAAC;IAiC3C;;OAEG;IACH,YAAY,IAAI,MAAM;IAItB;;OAEG;IACG,aAAa,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,MAAM,CAAC;IA2ElE;;OAEG;IACG,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAYlC;;OAEG;IACH,MAAM,CAAC,YAAY,IAAI,YAAY;CAOpC;AAMD;;;;;;;;;GASG;AACH,wBAAsB,aAAa,CAAC,OAAO,CAAC,EAAE;IAC5C,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,OAAO,CAAC,cAAc,CAAC,CAG1B;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,gBAAgB,CACpC,OAAO,CAAC,EAAE,kBAAkB,GAC3B,OAAO,CAAC,MAAM,CAAC,CAGjB;AAED;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,OAAO,CAAC,EAAE;IAC7C,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,OAAO,CAAC,IAAI,CAAC,CAGhB;AAED;;;;;;;;;GASG;AACH,wBAAgB,eAAe,IAAI,YAAY,CAE9C"}
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
import * as fs from "node:fs";
|
|
7
7
|
import * as os from "node:os";
|
|
8
8
|
import * as path from "node:path";
|
|
9
|
+
import { loadNodeLlamaCpp } from "./load-llama.js";
|
|
9
10
|
// Model configuration
|
|
10
11
|
// Using mradermacher's public mirror since jinaai's repo requires authentication
|
|
11
12
|
const MODEL_CONFIG = {
|
|
@@ -138,7 +139,7 @@ export class LLMManager {
|
|
|
138
139
|
totalSize: MODEL_CONFIG.size,
|
|
139
140
|
});
|
|
140
141
|
try {
|
|
141
|
-
const { createModelDownloader } = await
|
|
142
|
+
const { createModelDownloader } = await loadNodeLlamaCpp();
|
|
142
143
|
const downloader = await createModelDownloader({
|
|
143
144
|
modelUri: `hf:${MODEL_CONFIG.huggingFaceRepo}/${MODEL_CONFIG.fileName}`,
|
|
144
145
|
dirPath: path.dirname(targetPath),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm-manager.js","sourceRoot":"","sources":["../../src/converters/llm-manager.ts"],"names":[],"mappings":"AAAA,gCAAgC;AAEhC,4EAA4E;AAC5E,wEAAwE;AACxE,6EAA6E;AAC7E,4DAA4D;AAE5D,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"llm-manager.js","sourceRoot":"","sources":["../../src/converters/llm-manager.ts"],"names":[],"mappings":"AAAA,gCAAgC;AAEhC,4EAA4E;AAC5E,wEAAwE;AACxE,6EAA6E;AAC7E,4DAA4D;AAE5D,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAQlC,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAEnD,sBAAsB;AACtB,iFAAiF;AACjF,MAAM,YAAY,GAAG;IACnB,IAAI,EAAE,oBAAoB;IAC1B,eAAe,EAAE,+BAA+B;IAChD,QAAQ,EAAE,yBAAyB;IACnC,IAAI,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI,EAAE,mBAAmB;IAC7C,YAAY,EAAE,QAAQ;IACtB,WAAW,EAAE,OAAO;IACpB,OAAO,EAAE,KAAK;CACN,CAAC;AAEX,0CAA0C;AAC1C,yDAAyD;AACzD,MAAM,cAAc,GAAsB;IACxC;QACE,IAAI,EAAE,kBAAkB;QACxB,IAAI,EAAE,GAAG,GAAG,IAAI,GAAG,IAAI,EAAE,QAAQ;QACjC,YAAY,EAAE,MAAM;QACpB,WAAW,EAAE,OAAO;KACrB;IACD;QACE,IAAI,EAAE,oBAAoB;QAC1B,IAAI,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI,EAAE,SAAS;QACnC,YAAY,EAAE,QAAQ;QACtB,WAAW,EAAE,OAAO;KACrB;IACD;QACE,IAAI,EAAE,kBAAkB;QACxB,IAAI,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI,EAAE,SAAS;QACnC,YAAY,EAAE,MAAM;QACpB,WAAW,EAAE,OAAO;KACrB;CACF,CAAC;AAEF;;GAEG;AACH,SAAS,WAAW,CAAC,KAAa;IAChC,IAAI,KAAK,GAAG,IAAI;QAAE,OAAO,GAAG,KAAK,GAAG,CAAC;IACrC,IAAI,KAAK,GAAG,IAAI,GAAG,IAAI;QAAE,OAAO,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IACjE,IAAI,KAAK,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI;QAC5B,OAAO,GAAG,CAAC,KAAK,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IACnD,OAAO,GAAG,CAAC,KAAK,GAAG,CAAC,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;AAC1D,CAAC;AAED;;;GAGG;AACH,SAAS,kBAAkB;IACzB,OAAO,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;AACtD,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB;IAC1B,OAAO,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;AAChE,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,cAAc,CAAC,SAAiB;IAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACpC,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;AACpD,CAAC;AAED;;GAEG;AACH,MAAM,OAAO,UAAU;IACb,SAAS,CAAS;IAClB,aAAa,CAAoB;IAEzC,YAAY,OAA4D;QACtE,IAAI,CAAC,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,mBAAmB,EAAE,CAAC;QAC7D,IAAI,CAAC,aAAa,GAAG,OAAO,EAAE,OAAO,CAAC;IACxC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,IAAI,CAAC,KAAsC;QACvD,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACd,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;QAE7D,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAErD,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,KAAK,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBACrC,MAAM,IAAI,CAAC,IAAI,CAAC;oBACd,IAAI,EAAE,aAAa;oBACnB,MAAM,EAAE,OAAO;oBACf,IAAI,EAAE,IAAI,CAAC,SAAS;iBACrB,CAAC,CAAC;gBAEH,OAAO;oBACL,SAAS,EAAE,IAAI;oBACf,IAAI,EAAE,IAAI,CAAC,SAAS;oBACpB,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,aAAa,EAAE,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC;oBACtC,OAAO,EAAE,YAAY,CAAC,OAAO;iBAC9B,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,qBAAqB;QACvB,CAAC;QAED,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;QAE9D,OAAO;YACL,SAAS,EAAE,KAAK;YAChB,IAAI,EAAE,IAAI,CAAC,SAAS;SACrB,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,YAAY;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,OAA4B;QAC9C,MAAM,UAAU,GAAG,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC;QAExD,MAAM,cAAc,CAAC,UAAU,CAAC,CAAC;QAEjC,MAAM,IAAI,CAAC,IAAI,CAAC;YACd,IAAI,EAAE,gBAAgB;YACtB,SAAS,EAAE,YAAY,CAAC,IAAI;YAC5B,SAAS,EAAE,YAAY,CAAC,IAAI;SAC7B,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,EAAE,qBAAqB,EAAE,GAAG,MAAM,gBAAgB,EAAE,CAAC;YAC3D,MAAM,UAAU,GAAG,MAAM,qBAAqB,CAAC;gBAC7C,QAAQ,EAAE,MAAM,YAAY,CAAC,eAAe,IAAI,YAAY,CAAC,QAAQ,EAAE;gBACvE,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC;gBACjC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;gBACnC,UAAU,EAAE,CAAC,QAAQ,EAAE,EAAE;oBACvB,MAAM,UAAU,GAAG,QAAQ,CAAC,cAAc,CAAC;oBAC3C,MAAM,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC;oBACjC,MAAM,UAAU,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;oBAE9D,+BAA+B;oBAC/B,IAAI,KAAyB,CAAC;oBAC9B,IAAI,QAAQ,CAAC,cAAc,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;wBAClD,iEAAiE;wBACjE,KAAK,GAAG,SAAS,CAAC;oBACpB,CAAC;oBAED,qBAAqB;oBACrB,KAAK,IAAI,CAAC,IAAI,CAAC;wBACb,IAAI,EAAE,mBAAmB;wBACzB,UAAU;wBACV,KAAK;wBACL,UAAU;wBACV,KAAK;qBACN,CAAC,CAAC;oBAEH,uCAAuC;oBACvC,IAAI,OAAO,EAAE,UAAU,EAAE,CAAC;wBACxB,OAAO,CAAC,UAAU,CAAC,UAAU,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;oBACpD,CAAC;gBACH,CAAC;aACF,CAAC,CAAC;YAEH,qBAAqB;YACrB,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC;YAEjD,MAAM,IAAI,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,mBAAmB;gBACzB,IAAI,EAAE,YAAY;gBAClB,IAAI,EAAE,YAAY,CAAC,IAAI;aACxB,CAAC,CAAC;YAEH,IAAI,OAAO,EAAE,UAAU,EAAE,CAAC;gBACxB,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;YACnC,CAAC;YAED,OAAO,YAAY,CAAC;QACtB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAEtE,MAAM,IAAI,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,gBAAgB;gBACtB,KAAK,EAAE,GAAG;aACX,CAAC,CAAC;YAEH,IAAI,OAAO,EAAE,OAAO,EAAE,CAAC;gBACrB,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YACvB,CAAC;YAED,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW;QACf,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC3C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,+BAA+B;YAC/B,MAAM,GAAG,GAAG,KAA8B,CAAC;YAC3C,IAAI,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC1B,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,YAAY;QACjB,OAAO;YACL,WAAW,EAAE,mBAAmB,EAAE;YAClC,gBAAgB,EAAE,YAAY,CAAC,IAAI;YACnC,eAAe,EAAE,cAAc;SAChC,CAAC;IACJ,CAAC;CACF;AAED,+EAA+E;AAC/E,2CAA2C;AAC3C,+EAA+E;AAE/E;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,OAEnC;IACC,MAAM,OAAO,GAAG,IAAI,UAAU,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;IAClE,OAAO,OAAO,CAAC,UAAU,EAAE,CAAC;AAC9B,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,OAA4B;IAE5B,MAAM,OAAO,GAAG,IAAI,UAAU,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;IAClE,OAAO,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;AACxC,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAEpC;IACC,MAAM,OAAO,GAAG,IAAI,UAAU,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;IAClE,OAAO,OAAO,CAAC,WAAW,EAAE,CAAC;AAC/B,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,eAAe;IAC7B,OAAO,UAAU,CAAC,YAAY,EAAE,CAAC;AACnC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"load-llama.d.ts","sourceRoot":"","sources":["../../src/converters/load-llama.ts"],"names":[],"mappings":"AAQA,wBAAgB,gBAAgB,IAAI,OAAO,CAAC,cAAc,gBAAgB,CAAC,CAAC,CAqB3E"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
// `node-llama-cpp` is declared as an optional peer dependency in package.json
|
|
2
|
+
// because it carries hundreds of MB of platform-specific native binaries that
|
|
3
|
+
// consumers of the non-LLM APIs (`convertToMarkdown` etc.) don't need. This
|
|
4
|
+
// helper centralises the dynamic import so missing-peer failures surface a
|
|
5
|
+
// clear, actionable error instead of a generic ERR_MODULE_NOT_FOUND.
|
|
6
|
+
let cached = null;
|
|
7
|
+
export function loadNodeLlamaCpp() {
|
|
8
|
+
if (!cached) {
|
|
9
|
+
cached = import("node-llama-cpp").catch((err) => {
|
|
10
|
+
const code = err?.code;
|
|
11
|
+
const message = err?.message ?? "";
|
|
12
|
+
if (code === "ERR_MODULE_NOT_FOUND" ||
|
|
13
|
+
code === "MODULE_NOT_FOUND" ||
|
|
14
|
+
message.includes("node-llama-cpp")) {
|
|
15
|
+
throw new Error('get-md: LLM-based conversion requires the optional peer dependency "node-llama-cpp". ' +
|
|
16
|
+
"Install it alongside get-md (e.g. `pnpm add node-llama-cpp`) to enable LLMConverter, " +
|
|
17
|
+
"LLMManager, and `getmd --download-model`. The HTML→Markdown path (convertToMarkdown) " +
|
|
18
|
+
"works without it.");
|
|
19
|
+
}
|
|
20
|
+
throw err;
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
return cached;
|
|
24
|
+
}
|
|
25
|
+
//# sourceMappingURL=load-llama.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"load-llama.js","sourceRoot":"","sources":["../../src/converters/load-llama.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,8EAA8E;AAC9E,4EAA4E;AAC5E,2EAA2E;AAC3E,qEAAqE;AAErE,IAAI,MAAM,GAAoD,IAAI,CAAC;AAEnE,MAAM,UAAU,gBAAgB;IAC9B,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;YACvD,MAAM,IAAI,GAAI,GAAyC,EAAE,IAAI,CAAC;YAC9D,MAAM,OAAO,GAAI,GAAyB,EAAE,OAAO,IAAI,EAAE,CAAC;YAC1D,IACE,IAAI,KAAK,sBAAsB;gBAC/B,IAAI,KAAK,kBAAkB;gBAC3B,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAClC,CAAC;gBACD,MAAM,IAAI,KAAK,CACb,uFAAuF;oBACrF,uFAAuF;oBACvF,uFAAuF;oBACvF,mBAAmB,CACtB,CAAC;YACJ,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@nanocollective/get-md",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.4.0",
|
|
4
4
|
"description": "Fast HTML to Markdown converter optimized for LLM consumption",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -34,7 +34,7 @@
|
|
|
34
34
|
"test:lint": "biome lint .",
|
|
35
35
|
"test:lint:fix": "biome check --write .",
|
|
36
36
|
"test:knip": "knip",
|
|
37
|
-
"test:audit": "pnpm audit --audit-level=high",
|
|
37
|
+
"test:audit": "pnpm audit --audit-level=high --ignore-unfixable",
|
|
38
38
|
"test:security": "semgrep scan --config auto --error",
|
|
39
39
|
"format": "biome format --write .",
|
|
40
40
|
"prepublishOnly": "pnpm run build"
|
|
@@ -59,10 +59,17 @@
|
|
|
59
59
|
"cli-progress": "^3.12.0",
|
|
60
60
|
"commander": "^14.0.2",
|
|
61
61
|
"happy-dom-without-node": "^14.12.3",
|
|
62
|
-
"node-llama-cpp": "^3.15.0",
|
|
63
62
|
"turndown": "^7.2.2",
|
|
64
63
|
"turndown-plugin-gfm": "^1.0.2"
|
|
65
64
|
},
|
|
65
|
+
"peerDependencies": {
|
|
66
|
+
"node-llama-cpp": "^3.15.0"
|
|
67
|
+
},
|
|
68
|
+
"peerDependenciesMeta": {
|
|
69
|
+
"node-llama-cpp": {
|
|
70
|
+
"optional": true
|
|
71
|
+
}
|
|
72
|
+
},
|
|
66
73
|
"repository": {
|
|
67
74
|
"type": "git",
|
|
68
75
|
"url": "https://github.com/nano-collective/get-md.git"
|
|
@@ -78,6 +85,7 @@
|
|
|
78
85
|
"ava": "^6.4.1",
|
|
79
86
|
"c8": "^10.1.3",
|
|
80
87
|
"knip": "^5.80.2",
|
|
88
|
+
"node-llama-cpp": "^3.15.0",
|
|
81
89
|
"tsx": "^4.21.0",
|
|
82
90
|
"typescript": "^5.9.3"
|
|
83
91
|
},
|