@memberjunction/core-actions 6.1.0-edge.4 → 6.1.0-edge.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -1
- package/dist/config.d.ts +18 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +22 -11
- package/dist/config.js.map +1 -1
- package/dist/custom/integration/graphql-query.action.d.ts.map +1 -1
- package/dist/custom/integration/graphql-query.action.js +2 -1
- package/dist/custom/integration/graphql-query.action.js.map +1 -1
- package/dist/custom/search/scoped-search.action.d.ts +7 -0
- package/dist/custom/search/scoped-search.action.d.ts.map +1 -1
- package/dist/custom/search/scoped-search.action.js +156 -14
- package/dist/custom/search/scoped-search.action.js.map +1 -1
- package/dist/custom/utilities/base-file-handler.d.ts.map +1 -1
- package/dist/custom/utilities/base-file-handler.js +2 -0
- package/dist/custom/utilities/base-file-handler.js.map +1 -1
- package/dist/custom/web/rss-feed-parsing.d.ts +135 -0
- package/dist/custom/web/rss-feed-parsing.d.ts.map +1 -0
- package/dist/custom/web/rss-feed-parsing.js +271 -0
- package/dist/custom/web/rss-feed-parsing.js.map +1 -0
- package/dist/custom/web/rss-feed-read.action.d.ts +108 -0
- package/dist/custom/web/rss-feed-read.action.d.ts.map +1 -0
- package/dist/custom/web/rss-feed-read.action.js +325 -0
- package/dist/custom/web/rss-feed-read.action.js.map +1 -0
- package/dist/custom/web/tavily-search.action.d.ts +108 -0
- package/dist/custom/web/tavily-search.action.d.ts.map +1 -0
- package/dist/custom/web/tavily-search.action.js +275 -0
- package/dist/custom/web/tavily-search.action.js.map +1 -0
- package/dist/custom/web/url-metadata-extractor.action.d.ts.map +1 -1
- package/dist/custom/web/url-metadata-extractor.action.js +3 -1
- package/dist/custom/web/url-metadata-extractor.action.js.map +1 -1
- package/dist/custom/web/web-page-content.action.d.ts.map +1 -1
- package/dist/custom/web/web-page-content.action.js +5 -1
- package/dist/custom/web/web-page-content.action.js.map +1 -1
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/package.json +39 -39
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"base-file-handler.d.ts","sourceRoot":"","sources":["../../../src/custom/utilities/base-file-handler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;
|
|
1
|
+
{"version":3,"file":"base-file-handler.d.ts","sourceRoot":"","sources":["../../../src/custom/utilities/base-file-handler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAM/D;;;;;;GAMG;AACH,8BAAsB,qBAAsB,SAAQ,UAAU;IAC1D;;;;;;;;;OASG;cACa,cAAc,CAC1B,MAAM,EAAE,eAAe,EACvB,aAAa,EAAE,MAAM,EACrB,aAAa,GAAE,MAAiB,EAChC,YAAY,GAAE,MAAkB,GACjC,OAAO,CAAC;QACP,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC;QACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,SAAS,GAAG,KAAK,GAAG,QAAQ,CAAC;KACxC,CAAC;IAyBF;;OAEG;YACW,iBAAiB;IAkC/B;;OAEG;YACW,WAAW;IAiDzB;;;;;;;;;;;OAWG;cACa,eAAe,CAC3B,OAAO,EAAE,MAAM,GAAG,MAAM,EACxB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,eAAe,EACvB,kBAAkB,CAAC,EAAE,MAAM,EAC3B,WAAW,CAAC,EAAE,MAAM,GACrB,OAAO,CAAC,MAAM,CAAC;IAiClB;;OAEG;IACH,OAAO,CAAC,wBAAwB;IAQhC;;;OAGG;YACW,uBAAuB;IAarC;;OAEG;YACW,uBAAuB;IASrC;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,MAAM,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,GAAG,GAAG;IAKxE;;OAEG;IACH,SAAS,CAAC,eAAe,CAAC,MAAM,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,GAAE,OAAe,GAAG,OAAO;IAM7G;;OAEG;IACH,SAAS,CAAC,eAAe,CAAC,MAAM,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,GAAE,MAAU,GAAG,MAAM;CAM1G"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { BaseAction } from "@memberjunction/actions";
|
|
2
2
|
import { RunView } from "@memberjunction/core";
|
|
3
3
|
import { FileStorageEngine } from "@memberjunction/storage";
|
|
4
|
+
import { DrainResponseBody } from "@memberjunction/network-utils";
|
|
4
5
|
/**
|
|
5
6
|
* Base class for actions that handle file inputs from multiple sources
|
|
6
7
|
* Provides common functionality for loading files from:
|
|
@@ -81,6 +82,7 @@ export class BaseFileHandlerAction extends BaseAction {
|
|
|
81
82
|
}
|
|
82
83
|
const response = await fetch(url);
|
|
83
84
|
if (!response.ok) {
|
|
85
|
+
await DrainResponseBody(response);
|
|
84
86
|
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
85
87
|
}
|
|
86
88
|
const contentType = response.headers.get('content-type');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"base-file-handler.js","sourceRoot":"","sources":["../../../src/custom/utilities/base-file-handler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAErD,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAE/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"base-file-handler.js","sourceRoot":"","sources":["../../../src/custom/utilities/base-file-handler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAErD,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAE/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAElE;;;;;;GAMG;AACH,MAAM,OAAgB,qBAAsB,SAAQ,UAAU;IAC1D;;;;;;;;;OASG;IACO,KAAK,CAAC,cAAc,CAC1B,MAAuB,EACvB,aAAqB,EACrB,gBAAwB,QAAQ,EAChC,eAAuB,SAAS;QAOhC,sCAAsC;QACtC,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,aAAa,CAAC,WAAW,EAAE,CAAC,CAAC;QACzG,IAAI,WAAW,EAAE,KAAK,EAAE,CAAC;YACrB,OAAO,MAAM,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,KAAK,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC,CAAC;QAC9E,CAAC;QAED,oBAAoB;QACpB,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,YAAY,CAAC,WAAW,EAAE,CAAC,CAAC;QACzG,IAAI,YAAY,EAAE,KAAK,EAAE,CAAC;YACtB,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;QACjE,CAAC;QAED,wBAAwB;QACxB,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,aAAa,CAAC,WAAW,EAAE,CAAC,CAAC;QACvG,IAAI,SAAS,EAAE,KAAK,EAAE,CAAC;YACnB,OAAO;gBACH,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,QAAQ,EAAE;gBACnC,MAAM,EAAE,QAAQ;aACnB,CAAC;QACN,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,aAAa,KAAK,YAAY,QAAQ,aAAa,EAAE,CAAC,CAAC;IAChH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,iBAAiB,CAAC,MAAc,EAAE,MAAuB;QAMnE,IAAI,CAAC;YACD,MAAM,EAAE,GAAG,IAAI,OAAO,EAAE,CAAC;YACzB,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,OAAO,CAAe;gBAC9C,UAAU,EAAE,WAAW;gBACvB,WAAW,EAAE,SAAS,MAAM,GAAG;gBAC/B,UAAU,EAAE,eAAe;aAC9B,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;YAEvB,IAAI,CAAC,UAAU,CAAC,OAAO,IAAI,UAAU,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzD,MAAM,IAAI,KAAK,CAAC,gCAAgC,MAAM,EAAE,CAAC,CAAC;YAC9D,CAAC;YAED,MAAM,IAAI,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACnC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAChE,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,IAAI,CAAC;YACjD,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,CAAC;YAEjE,OAAO;gBACH,OAAO;gBACP,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,QAAQ,EAAE,IAAI,CAAC,WAAW,IAAI,SAAS;gBACvC,MAAM,EAAE,SAAS;aACpB,CAAC;QACN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,qCAAqC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACnH,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,WAAW,CAAC,GAAW;QAMjC,IAAI,CAAC;YACD,eAAe;YACf,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACjD,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;YAC9D,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACf,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAC;gBAClC,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YACvE,CAAC;YAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;YACzD,MAAM,kBAAkB,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;YACvE,IAAI,QAA4B,CAAC;YAEjC,yDAAyD;YACzD,IAAI,kBAAkB,EAAE,CAAC;gBACrB,MAAM,KAAK,GAAG,kBAAkB,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAC;gBACjF,IAAI,KAAK,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;oBACpB,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;gBAC7C,CAAC;YACL,CAAC;YAED,+CAA+C;YAC/C,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACZ,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;YAChD,CAAC;YAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAEtC,OAAO;gBACH,OAAO;gBACP,QAAQ;gBACR,QAAQ,EAAE,WAAW,IAAI,SAAS;gBAClC,MAAM,EAAE,KAAK;aAChB,CAAC;QACN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,iCAAiC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC/G,CAAC;IACL,CAAC;IAED;;;;;;;;;;;OAWG;IACO,KAAK,CAAC,eAAe,CAC3B,OAAwB,EACxB,QAAgB,EAChB,QAAgB,EAChB,MAAuB,EACvB,kBAA2B,EAC3B,WAAoB;QAEpB,IAAI,CAAC;YACD,MAAM,iBAAiB,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;YAEnE,6BAA6B;YAC7B,IAAI,gBAAoC,CAAC;YACzC,IAAI,kBAAkB,EAAE,CAAC;gBACrB,MAAM,OAAO,GAAG,IAAI,CAAC,wBAAwB,CAAC,kBAAkB,CAAC,CAAC;gBAClE,gBAAgB,GAAG,OAAO,CAAC,EAAE,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACJ,gBAAgB,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC;YAClE,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACzE,MAAM,UAAU,GAAG,WAAW;gBAC1B,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;gBACjC,CAAC,CAAC,SAAS,CAAC;YAEhB,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,QAAQ,CAAC,UAAU,CAAC;gBACvD,OAAO,EAAE,MAAM;gBACf,QAAQ;gBACR,QAAQ;gBACR,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,gBAAgB;gBAChB,UAAU;aACb,CAAC,CAAC;YAEH,OAAO,MAAM,CAAC,MAAM,CAAC;QACzB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,mCAAmC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACjH,CAAC;IACL,CAAC;IAED;;OAEG;IACK,wBAAwB,CAAC,WAAmB;QAChD,MAAM,OAAO,GAAG,iBAAiB,CAAC,QAAQ,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;QACzE,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CAAC,uBAAuB,WAAW,iDAAiD,CAAC,CAAC;QACzG,CAAC;QACD,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,uBAAuB,CAAC,MAAuB;QACzD,gFAAgF;QAChF,MAAM,OAAO,GAAG,MAAM,CAAC,OAA8C,CAAC;QACtE,MAAM,UAAU,GAAG,OAAO,EAAE,0BAA0B,CAAC;QACvD,IAAI,UAAU,EAAE,CAAC;YACb,OAAO,UAAU,CAAC,QAAQ,EAAE,CAAC;QACjC,CAAC;QAED,yCAAyC;QACzC,MAAM,QAAQ,GAAG,iBAAiB,CAAC,QAAQ,CAAC,qBAAqB,EAAE,CAAC;QACpE,OAAO,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC;IAChC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,uBAAuB,CAAC,IAAkB,EAAE,MAAuB;QAC7E,MAAM,QAAQ,GAAG,iBAAiB,CAAC,QAAQ,CAAC,uBAAuB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACrF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,8CAA8C,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QACrF,CAAC;QAED,OAAO,iBAAiB,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;IACpF,CAAC;IAED;;OAEG;IACO,aAAa,CAAC,MAAuB,EAAE,SAAiB;QAC9D,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC;QAC/F,OAAO,KAAK,EAAE,KAAK,CAAC;IACxB,CAAC;IAED;;OAEG;IACO,eAAe,CAAC,MAAuB,EAAE,SAAiB,EAAE,eAAwB,KAAK;QAC/F,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QACpD,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,YAAY,CAAC;QAC/D,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC;IAClD,CAAC;IAED;;OAEG;IACO,eAAe,CAAC,MAAuB,EAAE,SAAiB,EAAE,eAAuB,CAAC;QAC1F,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QACpD,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,YAAY,CAAC;QAC/D,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC1B,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC;IAC3C,CAAC;CACJ"}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RSS and Atom feed parsing and keyword scoring — pure functions.
|
|
3
|
+
*
|
|
4
|
+
* No HTTP, no logging, no clock of its own: every function that needs the current
|
|
5
|
+
* time takes it as a parameter. That is deliberate. A single feed read filters by
|
|
6
|
+
* age and then scores by recency, and if those two steps read `Date.now()`
|
|
7
|
+
* independently they can disagree — an article can pass the age filter and then
|
|
8
|
+
* score as out-of-window, producing a result the caller cannot explain. One
|
|
9
|
+
* injected `now` also makes every threshold here testable without faking timers.
|
|
10
|
+
*
|
|
11
|
+
* Parsing is by regex rather than by an XML parser. That is a deliberate trade:
|
|
12
|
+
* feeds in the wild are frequently not well-formed, and a strict parser fails the
|
|
13
|
+
* whole document over one unescaped ampersand in one item. The regex approach
|
|
14
|
+
* degrades per item instead. The cost is that deeply nested or namespace-heavy
|
|
15
|
+
* markup is not understood; both are rare in the item-level fields extracted here.
|
|
16
|
+
*/
|
|
17
|
+
/** One article from a feed, normalized across RSS 2.0 and Atom. */
|
|
18
|
+
export interface FeedArticle {
|
|
19
|
+
title: string;
|
|
20
|
+
link: string;
|
|
21
|
+
/** Plain text — HTML tags stripped and entities decoded. */
|
|
22
|
+
description: string;
|
|
23
|
+
/**
|
|
24
|
+
* ISO 8601 publication date, or null when the feed did not carry a parseable
|
|
25
|
+
* one. Null is preserved rather than defaulted to "now", because a
|
|
26
|
+
* fabricated date would make an undated item the freshest thing in the batch.
|
|
27
|
+
*/
|
|
28
|
+
publishedAt: string | null;
|
|
29
|
+
categories: string[];
|
|
30
|
+
/** Which feed this came from, so a merged result set stays attributable. */
|
|
31
|
+
feedName: string;
|
|
32
|
+
}
|
|
33
|
+
/** An article with its keyword-relevance and recency scores. */
|
|
34
|
+
export interface ScoredFeedArticle {
|
|
35
|
+
article: FeedArticle;
|
|
36
|
+
/** Weighted sum of keyword hits: title 3, category 2, description 1, per keyword. */
|
|
37
|
+
relevanceScore: number;
|
|
38
|
+
/** 1.0 for something published now, decaying to 0.1 at the edge of the window; 0 for undated. */
|
|
39
|
+
recencyScore: number;
|
|
40
|
+
/** `relevanceScore * 10 + recencyScore * 5` — relevance dominates, recency breaks ties. */
|
|
41
|
+
totalScore: number;
|
|
42
|
+
matchedKeywords: string[];
|
|
43
|
+
}
|
|
44
|
+
/** Weights behind {@link computeRelevanceScore}, exported so callers can explain a score. */
|
|
45
|
+
export declare const RELEVANCE_WEIGHTS: {
|
|
46
|
+
readonly title: 3;
|
|
47
|
+
readonly category: 2;
|
|
48
|
+
readonly description: 1;
|
|
49
|
+
};
|
|
50
|
+
/**
|
|
51
|
+
* Reduce feed text to plain text: decode, strip markup, decode again, collapse.
|
|
52
|
+
*
|
|
53
|
+
* Decoding runs *before* stripping because RSS descriptions overwhelmingly carry
|
|
54
|
+
* their HTML escaped — `<p>text</p>` rather than `<p>text</p>`. Strip
|
|
55
|
+
* first and those tags survive as literal `<p>` in the output, which is what a
|
|
56
|
+
* consumer then has to clean up itself.
|
|
57
|
+
*
|
|
58
|
+
* The second decode pass exists for that same escaped-HTML case: source that
|
|
59
|
+
* renders as `&` arrives as `&amp;`, so one pass leaves a visible `&`
|
|
60
|
+
* where the reader expects `&`. Running it again after the tags are gone finishes
|
|
61
|
+
* the job. Text that was never double-escaped has no entities left by then, so the
|
|
62
|
+
* extra pass is a no-op on it.
|
|
63
|
+
*/
|
|
64
|
+
export declare function stripHtml(text: string): string;
|
|
65
|
+
/**
|
|
66
|
+
* Parse a feed document into articles. Handles RSS 2.0 (`<item>`) and Atom
|
|
67
|
+
* (`<entry>`); a document containing both has both read, since some publishers
|
|
68
|
+
* serve a hybrid.
|
|
69
|
+
*
|
|
70
|
+
* Items with no title are skipped — there is nothing to show or score. Items with
|
|
71
|
+
* no parseable date are kept with `publishedAt: null`, which is what lets the
|
|
72
|
+
* caller report them rather than wonder where they went. Age filtering is a
|
|
73
|
+
* separate step ({@link filterByAge}) precisely so that decision stays visible.
|
|
74
|
+
*/
|
|
75
|
+
export declare function parseFeedArticles(xml: string, feedName: string): FeedArticle[];
|
|
76
|
+
/** A feed date as ISO 8601, or null when it is missing or unparseable. */
|
|
77
|
+
export declare function parseFeedDate(raw: string): string | null;
|
|
78
|
+
/** The outcome of an age filter, keeping what it dropped countable. */
|
|
79
|
+
export interface AgeFilterResult {
|
|
80
|
+
kept: FeedArticle[];
|
|
81
|
+
/** Dropped for being older than the window. */
|
|
82
|
+
tooOldCount: number;
|
|
83
|
+
/** Undated articles, kept or dropped per `includeUndated`. */
|
|
84
|
+
undatedCount: number;
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Keep articles published within `maxAgeDays` of `now`.
|
|
88
|
+
*
|
|
89
|
+
* Undated articles cannot be age-filtered, so `includeUndated` decides them
|
|
90
|
+
* explicitly rather than having them silently vanish — which is what happens when
|
|
91
|
+
* an undated item is compared against a cutoff. Both counts come back so the
|
|
92
|
+
* caller can say how many articles a quiet result set actually hid.
|
|
93
|
+
*
|
|
94
|
+
* A future-dated article is kept: publishers do post-date, and a clock skew of a
|
|
95
|
+
* few minutes should not delete the newest item in the feed.
|
|
96
|
+
*/
|
|
97
|
+
export declare function filterByAge(articles: FeedArticle[], maxAgeDays: number, now: Date, includeUndated: boolean): AgeFilterResult;
|
|
98
|
+
/**
|
|
99
|
+
* Score keyword relevance. A hit in the title counts 3, in a category 2, in the
|
|
100
|
+
* description 1, and one keyword can hit all three.
|
|
101
|
+
*
|
|
102
|
+
* Matching is case-insensitive substring, which is what makes it useful on feed
|
|
103
|
+
* text — but it means 'AI' matches 'said'. Prefer multi-word keywords when that
|
|
104
|
+
* matters; the alternative, word-boundary matching, would miss 'AI-driven' and
|
|
105
|
+
* plurals, which costs more in practice.
|
|
106
|
+
*/
|
|
107
|
+
export declare function computeRelevanceScore(article: FeedArticle, keywords: string[]): {
|
|
108
|
+
score: number;
|
|
109
|
+
matched: string[];
|
|
110
|
+
};
|
|
111
|
+
/**
|
|
112
|
+
* Recency as a 0-1 decay across the window: 1.0 for something published at `now`,
|
|
113
|
+
* floored at 0.1 at the far edge so an in-window article never scores as though it
|
|
114
|
+
* were out of window. Undated and out-of-window both score 0.
|
|
115
|
+
*/
|
|
116
|
+
export declare function computeRecencyScore(publishedAt: string | null, timeWindowDays: number, now: Date): number;
|
|
117
|
+
/**
|
|
118
|
+
* Score every article and rank them, highest total first.
|
|
119
|
+
*
|
|
120
|
+
* With no keywords, relevance is 0 for everything and the ranking is pure
|
|
121
|
+
* recency — which is the right behaviour for "just show me what is new" rather
|
|
122
|
+
* than an error.
|
|
123
|
+
*/
|
|
124
|
+
export declare function scoreAndRankArticles(articles: FeedArticle[], keywords: string[], timeWindowDays: number, now: Date): ScoredFeedArticle[];
|
|
125
|
+
/**
|
|
126
|
+
* Drop articles that matched nothing and are not fresh enough to be interesting
|
|
127
|
+
* on their own.
|
|
128
|
+
*
|
|
129
|
+
* The recency escape hatch exists because a keyword list is always incomplete: an
|
|
130
|
+
* article published in the last quarter of the window is worth surfacing even
|
|
131
|
+
* though none of the supplied keywords appeared in it. With no keywords supplied,
|
|
132
|
+
* everything is relevant by definition and nothing is dropped.
|
|
133
|
+
*/
|
|
134
|
+
export declare function filterRelevant(scored: ScoredFeedArticle[], keywords: string[], recencyFloor?: number): ScoredFeedArticle[];
|
|
135
|
+
//# sourceMappingURL=rss-feed-parsing.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rss-feed-parsing.d.ts","sourceRoot":"","sources":["../../../src/custom/web/rss-feed-parsing.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,mEAAmE;AACnE,MAAM,WAAW,WAAW;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,4DAA4D;IAC5D,WAAW,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,4EAA4E;IAC5E,QAAQ,EAAE,MAAM,CAAC;CACpB;AAED,gEAAgE;AAChE,MAAM,WAAW,iBAAiB;IAC9B,OAAO,EAAE,WAAW,CAAC;IACrB,qFAAqF;IACrF,cAAc,EAAE,MAAM,CAAC;IACvB,iGAAiG;IACjG,YAAY,EAAE,MAAM,CAAC;IACrB,2FAA2F;IAC3F,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,EAAE,CAAC;CAC7B;AAID,6FAA6F;AAC7F,eAAO,MAAM,iBAAiB;;;;CAAqD,CAAC;AA0DpF;;;;;;;;;;;;;GAaG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAI9C;AAED;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,WAAW,EAAE,CAkD9E;AAQD,0EAA0E;AAC1E,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAIxD;AAID,uEAAuE;AACvE,MAAM,WAAW,eAAe;IAC5B,IAAI,EAAE,WAAW,EAAE,CAAC;IACpB,+CAA+C;IAC/C,WAAW,EAAE,MAAM,CAAC;IACpB,8DAA8D;IAC9D,YAAY,EAAE,MAAM,CAAC;CACxB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,WAAW,CACvB,QAAQ,EAAE,WAAW,EAAE,EACvB,UAAU,EAAE,MAAM,EAClB,GAAG,EAAE,IAAI,EACT,cAAc,EAAE,OAAO,GACxB,eAAe,CAoBjB;AAID;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,EAAE,CAAA;CAAE,CAwBpH;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,GAAG,MAAM,CAOzG;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAChC,QAAQ,EAAE,WAAW,EAAE,EACvB,QAAQ,EAAE,MAAM,EAAE,EAClB,cAAc,EAAE,MAAM,EACtB,GAAG,EAAE,IAAI,GACV,iBAAiB,EAAE,CAgBrB;AAED;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,iBAAiB,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,YAAY,SAAO,GAAG,iBAAiB,EAAE,CAGxH"}
|
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RSS and Atom feed parsing and keyword scoring — pure functions.
|
|
3
|
+
*
|
|
4
|
+
* No HTTP, no logging, no clock of its own: every function that needs the current
|
|
5
|
+
* time takes it as a parameter. That is deliberate. A single feed read filters by
|
|
6
|
+
* age and then scores by recency, and if those two steps read `Date.now()`
|
|
7
|
+
* independently they can disagree — an article can pass the age filter and then
|
|
8
|
+
* score as out-of-window, producing a result the caller cannot explain. One
|
|
9
|
+
* injected `now` also makes every threshold here testable without faking timers.
|
|
10
|
+
*
|
|
11
|
+
* Parsing is by regex rather than by an XML parser. That is a deliberate trade:
|
|
12
|
+
* feeds in the wild are frequently not well-formed, and a strict parser fails the
|
|
13
|
+
* whole document over one unescaped ampersand in one item. The regex approach
|
|
14
|
+
* degrades per item instead. The cost is that deeply nested or namespace-heavy
|
|
15
|
+
* markup is not understood; both are rare in the item-level fields extracted here.
|
|
16
|
+
*/
|
|
17
|
+
const MS_PER_DAY = 24 * 60 * 60 * 1000;
|
|
18
|
+
/** Weights behind {@link computeRelevanceScore}, exported so callers can explain a score. */
|
|
19
|
+
export const RELEVANCE_WEIGHTS = { title: 3, category: 2, description: 1 };
|
|
20
|
+
/** How long a description is kept. Feeds sometimes inline a whole article body. */
|
|
21
|
+
const MAX_DESCRIPTION_LENGTH = 500;
|
|
22
|
+
// ─── Parsing ──────────────────────────────────────────────────────────────────
|
|
23
|
+
/**
|
|
24
|
+
* Extract one tag's text, handling CDATA.
|
|
25
|
+
*
|
|
26
|
+
* The name is matched only when followed by whitespace, `/` or `>`, so asking for
|
|
27
|
+
* `link` does not match `<linkGroup>` — and, more importantly for Atom, asking for
|
|
28
|
+
* `title` does not match a namespaced sibling that merely starts with it.
|
|
29
|
+
*/
|
|
30
|
+
function extractTag(xml, tagName) {
|
|
31
|
+
const pattern = new RegExp(`<${tagName}(?:\\s[^>]*)?>(?:<!\\[CDATA\\[)?([\\s\\S]*?)(?:\\]\\]>)?<\\/${tagName}\\s*>`);
|
|
32
|
+
return pattern.exec(xml)?.[1]?.trim() ?? '';
|
|
33
|
+
}
|
|
34
|
+
/** Every occurrence of a tag's text, in document order. */
|
|
35
|
+
function extractAllTags(xml, tagName) {
|
|
36
|
+
const pattern = new RegExp(`<${tagName}(?:\\s[^>]*)?>(?:<!\\[CDATA\\[)?([\\s\\S]*?)(?:\\]\\]>)?<\\/${tagName}\\s*>`, 'g');
|
|
37
|
+
const results = [];
|
|
38
|
+
let match;
|
|
39
|
+
while ((match = pattern.exec(xml)) !== null) {
|
|
40
|
+
const value = decodeEntities(match[1]?.trim() ?? '');
|
|
41
|
+
if (value)
|
|
42
|
+
results.push(value);
|
|
43
|
+
}
|
|
44
|
+
return results;
|
|
45
|
+
}
|
|
46
|
+
/** One attribute off the first matching tag, e.g. Atom's `<link href="…"/>`. */
|
|
47
|
+
function extractAttribute(xml, tagName, attribute) {
|
|
48
|
+
const pattern = new RegExp(`<${tagName}(?:\\s[^>]*)?\\s${attribute}\\s*=\\s*["']([^"']*)["']`);
|
|
49
|
+
return pattern.exec(xml)?.[1]?.trim() ?? '';
|
|
50
|
+
}
|
|
51
|
+
/** Decode the entities that actually appear in feed text. */
|
|
52
|
+
function decodeEntities(text) {
|
|
53
|
+
return text
|
|
54
|
+
.replace(/&#x([0-9a-fA-F]+);/g, (_, hex) => safeCodePoint(parseInt(hex, 16)))
|
|
55
|
+
.replace(/&#(\d+);/g, (_, dec) => safeCodePoint(parseInt(dec, 10)))
|
|
56
|
+
.replace(/</g, '<')
|
|
57
|
+
.replace(/>/g, '>')
|
|
58
|
+
.replace(/"/g, '"')
|
|
59
|
+
.replace(/'/g, "'")
|
|
60
|
+
.replace(/ /g, ' ')
|
|
61
|
+
// `&` is decoded last so `&lt;` yields the literal text `<`
|
|
62
|
+
// rather than being decoded twice into `<`.
|
|
63
|
+
.replace(/&/g, '&');
|
|
64
|
+
}
|
|
65
|
+
function safeCodePoint(code) {
|
|
66
|
+
// An out-of-range numeric entity throws from fromCodePoint; dropping it is
|
|
67
|
+
// better than failing the whole item over one bad character reference.
|
|
68
|
+
return Number.isFinite(code) && code >= 0 && code <= 0x10ffff ? String.fromCodePoint(code) : '';
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Reduce feed text to plain text: decode, strip markup, decode again, collapse.
|
|
72
|
+
*
|
|
73
|
+
* Decoding runs *before* stripping because RSS descriptions overwhelmingly carry
|
|
74
|
+
* their HTML escaped — `<p>text</p>` rather than `<p>text</p>`. Strip
|
|
75
|
+
* first and those tags survive as literal `<p>` in the output, which is what a
|
|
76
|
+
* consumer then has to clean up itself.
|
|
77
|
+
*
|
|
78
|
+
* The second decode pass exists for that same escaped-HTML case: source that
|
|
79
|
+
* renders as `&` arrives as `&amp;`, so one pass leaves a visible `&`
|
|
80
|
+
* where the reader expects `&`. Running it again after the tags are gone finishes
|
|
81
|
+
* the job. Text that was never double-escaped has no entities left by then, so the
|
|
82
|
+
* extra pass is a no-op on it.
|
|
83
|
+
*/
|
|
84
|
+
export function stripHtml(text) {
|
|
85
|
+
const unescaped = decodeEntities(text);
|
|
86
|
+
const withoutTags = unescaped.replace(/<[^>]*>/g, '');
|
|
87
|
+
return decodeEntities(withoutTags).replace(/\s+/g, ' ').trim();
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Parse a feed document into articles. Handles RSS 2.0 (`<item>`) and Atom
|
|
91
|
+
* (`<entry>`); a document containing both has both read, since some publishers
|
|
92
|
+
* serve a hybrid.
|
|
93
|
+
*
|
|
94
|
+
* Items with no title are skipped — there is nothing to show or score. Items with
|
|
95
|
+
* no parseable date are kept with `publishedAt: null`, which is what lets the
|
|
96
|
+
* caller report them rather than wonder where they went. Age filtering is a
|
|
97
|
+
* separate step ({@link filterByAge}) precisely so that decision stays visible.
|
|
98
|
+
*/
|
|
99
|
+
export function parseFeedArticles(xml, feedName) {
|
|
100
|
+
const articles = [];
|
|
101
|
+
const blockPattern = /<(item|entry)(?:\s[^>]*)?>([\s\S]*?)<\/\1\s*>/g;
|
|
102
|
+
let match;
|
|
103
|
+
while ((match = blockPattern.exec(xml)) !== null) {
|
|
104
|
+
const itemXml = match[2];
|
|
105
|
+
const title = stripHtml(extractTag(itemXml, 'title'));
|
|
106
|
+
if (!title)
|
|
107
|
+
continue;
|
|
108
|
+
// RSS puts the URL in <link>text</link>; Atom puts it in <link href="…"/>,
|
|
109
|
+
// which leaves the text extraction empty.
|
|
110
|
+
const link = decodeEntities(extractTag(itemXml, 'link')) ||
|
|
111
|
+
extractAttribute(itemXml, 'link', 'href') ||
|
|
112
|
+
extractPermalinkGuid(itemXml);
|
|
113
|
+
// RSS: description or content:encoded. Atom: summary or content.
|
|
114
|
+
const rawDescription = extractTag(itemXml, 'description') ||
|
|
115
|
+
extractTag(itemXml, 'content:encoded') ||
|
|
116
|
+
extractTag(itemXml, 'summary') ||
|
|
117
|
+
extractTag(itemXml, 'content');
|
|
118
|
+
// RSS: pubDate (RFC 822), sometimes dc:date. Atom: published, else updated.
|
|
119
|
+
const rawDate = extractTag(itemXml, 'pubDate') ||
|
|
120
|
+
extractTag(itemXml, 'published') ||
|
|
121
|
+
extractTag(itemXml, 'updated') ||
|
|
122
|
+
extractTag(itemXml, 'dc:date');
|
|
123
|
+
// Atom categories are an attribute, not element text.
|
|
124
|
+
const categories = extractAllTags(itemXml, 'category');
|
|
125
|
+
if (categories.length === 0) {
|
|
126
|
+
const term = extractAttribute(itemXml, 'category', 'term');
|
|
127
|
+
if (term)
|
|
128
|
+
categories.push(term);
|
|
129
|
+
}
|
|
130
|
+
articles.push({
|
|
131
|
+
title,
|
|
132
|
+
link,
|
|
133
|
+
description: stripHtml(rawDescription).slice(0, MAX_DESCRIPTION_LENGTH),
|
|
134
|
+
publishedAt: parseFeedDate(rawDate),
|
|
135
|
+
categories,
|
|
136
|
+
feedName,
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
return articles;
|
|
140
|
+
}
|
|
141
|
+
/** Some feeds carry the canonical URL as a permalink `<guid>` and no `<link>`. */
|
|
142
|
+
function extractPermalinkGuid(xml) {
|
|
143
|
+
const match = xml.match(/<guid[^>]*isPermaLink\s*=\s*["']true["'][^>]*>([\s\S]*?)<\/guid\s*>/);
|
|
144
|
+
return decodeEntities(match?.[1]?.trim() ?? '');
|
|
145
|
+
}
|
|
146
|
+
/** A feed date as ISO 8601, or null when it is missing or unparseable. */
|
|
147
|
+
export function parseFeedDate(raw) {
|
|
148
|
+
if (!raw)
|
|
149
|
+
return null;
|
|
150
|
+
const parsed = new Date(raw);
|
|
151
|
+
return isNaN(parsed.getTime()) ? null : parsed.toISOString();
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Keep articles published within `maxAgeDays` of `now`.
|
|
155
|
+
*
|
|
156
|
+
* Undated articles cannot be age-filtered, so `includeUndated` decides them
|
|
157
|
+
* explicitly rather than having them silently vanish — which is what happens when
|
|
158
|
+
* an undated item is compared against a cutoff. Both counts come back so the
|
|
159
|
+
* caller can say how many articles a quiet result set actually hid.
|
|
160
|
+
*
|
|
161
|
+
* A future-dated article is kept: publishers do post-date, and a clock skew of a
|
|
162
|
+
* few minutes should not delete the newest item in the feed.
|
|
163
|
+
*/
|
|
164
|
+
export function filterByAge(articles, maxAgeDays, now, includeUndated) {
|
|
165
|
+
const cutoff = now.getTime() - maxAgeDays * MS_PER_DAY;
|
|
166
|
+
const kept = [];
|
|
167
|
+
let tooOldCount = 0;
|
|
168
|
+
let undatedCount = 0;
|
|
169
|
+
for (const article of articles) {
|
|
170
|
+
if (article.publishedAt === null) {
|
|
171
|
+
undatedCount++;
|
|
172
|
+
if (includeUndated)
|
|
173
|
+
kept.push(article);
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
if (new Date(article.publishedAt).getTime() < cutoff) {
|
|
177
|
+
tooOldCount++;
|
|
178
|
+
continue;
|
|
179
|
+
}
|
|
180
|
+
kept.push(article);
|
|
181
|
+
}
|
|
182
|
+
return { kept, tooOldCount, undatedCount };
|
|
183
|
+
}
|
|
184
|
+
// ─── Scoring ──────────────────────────────────────────────────────────────────
|
|
185
|
+
/**
|
|
186
|
+
* Score keyword relevance. A hit in the title counts 3, in a category 2, in the
|
|
187
|
+
* description 1, and one keyword can hit all three.
|
|
188
|
+
*
|
|
189
|
+
* Matching is case-insensitive substring, which is what makes it useful on feed
|
|
190
|
+
* text — but it means 'AI' matches 'said'. Prefer multi-word keywords when that
|
|
191
|
+
* matters; the alternative, word-boundary matching, would miss 'AI-driven' and
|
|
192
|
+
* plurals, which costs more in practice.
|
|
193
|
+
*/
|
|
194
|
+
export function computeRelevanceScore(article, keywords) {
|
|
195
|
+
const title = article.title.toLowerCase();
|
|
196
|
+
const description = article.description.toLowerCase();
|
|
197
|
+
const categories = article.categories.map(c => c.toLowerCase()).join(' ');
|
|
198
|
+
let score = 0;
|
|
199
|
+
const matched = [];
|
|
200
|
+
for (const keyword of keywords) {
|
|
201
|
+
const needle = keyword.trim().toLowerCase();
|
|
202
|
+
if (needle.length === 0)
|
|
203
|
+
continue;
|
|
204
|
+
let keywordScore = 0;
|
|
205
|
+
if (title.includes(needle))
|
|
206
|
+
keywordScore += RELEVANCE_WEIGHTS.title;
|
|
207
|
+
if (categories.includes(needle))
|
|
208
|
+
keywordScore += RELEVANCE_WEIGHTS.category;
|
|
209
|
+
if (description.includes(needle))
|
|
210
|
+
keywordScore += RELEVANCE_WEIGHTS.description;
|
|
211
|
+
if (keywordScore > 0) {
|
|
212
|
+
score += keywordScore;
|
|
213
|
+
matched.push(keyword);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
return { score, matched };
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* Recency as a 0-1 decay across the window: 1.0 for something published at `now`,
|
|
220
|
+
* floored at 0.1 at the far edge so an in-window article never scores as though it
|
|
221
|
+
* were out of window. Undated and out-of-window both score 0.
|
|
222
|
+
*/
|
|
223
|
+
export function computeRecencyScore(publishedAt, timeWindowDays, now) {
|
|
224
|
+
if (publishedAt === null || timeWindowDays <= 0)
|
|
225
|
+
return 0;
|
|
226
|
+
const ageDays = (now.getTime() - new Date(publishedAt).getTime()) / MS_PER_DAY;
|
|
227
|
+
if (ageDays > timeWindowDays)
|
|
228
|
+
return 0;
|
|
229
|
+
// Future-dated articles clamp to the maximum rather than exceeding it.
|
|
230
|
+
if (ageDays <= 0)
|
|
231
|
+
return 1;
|
|
232
|
+
return Math.max(0.1, 1 - ageDays / timeWindowDays);
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* Score every article and rank them, highest total first.
|
|
236
|
+
*
|
|
237
|
+
* With no keywords, relevance is 0 for everything and the ranking is pure
|
|
238
|
+
* recency — which is the right behaviour for "just show me what is new" rather
|
|
239
|
+
* than an error.
|
|
240
|
+
*/
|
|
241
|
+
export function scoreAndRankArticles(articles, keywords, timeWindowDays, now) {
|
|
242
|
+
const scored = articles.map((article) => {
|
|
243
|
+
const { score: relevanceScore, matched } = computeRelevanceScore(article, keywords);
|
|
244
|
+
const recencyScore = computeRecencyScore(article.publishedAt, timeWindowDays, now);
|
|
245
|
+
return {
|
|
246
|
+
article,
|
|
247
|
+
relevanceScore,
|
|
248
|
+
recencyScore,
|
|
249
|
+
totalScore: relevanceScore * 10 + recencyScore * 5,
|
|
250
|
+
matchedKeywords: matched,
|
|
251
|
+
};
|
|
252
|
+
});
|
|
253
|
+
// Stable within equal scores: sort is stable in every supported runtime, so
|
|
254
|
+
// ties keep feed order and a re-read of an unchanged feed ranks identically.
|
|
255
|
+
return scored.sort((a, b) => b.totalScore - a.totalScore);
|
|
256
|
+
}
|
|
257
|
+
/**
|
|
258
|
+
* Drop articles that matched nothing and are not fresh enough to be interesting
|
|
259
|
+
* on their own.
|
|
260
|
+
*
|
|
261
|
+
* The recency escape hatch exists because a keyword list is always incomplete: an
|
|
262
|
+
* article published in the last quarter of the window is worth surfacing even
|
|
263
|
+
* though none of the supplied keywords appeared in it. With no keywords supplied,
|
|
264
|
+
* everything is relevant by definition and nothing is dropped.
|
|
265
|
+
*/
|
|
266
|
+
export function filterRelevant(scored, keywords, recencyFloor = 0.75) {
|
|
267
|
+
if (keywords.length === 0)
|
|
268
|
+
return scored;
|
|
269
|
+
return scored.filter(s => s.relevanceScore > 0 || s.recencyScore >= recencyFloor);
|
|
270
|
+
}
|
|
271
|
+
//# sourceMappingURL=rss-feed-parsing.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rss-feed-parsing.js","sourceRoot":"","sources":["../../../src/custom/web/rss-feed-parsing.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AA+BH,MAAM,UAAU,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAEvC,6FAA6F;AAC7F,MAAM,CAAC,MAAM,iBAAiB,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAW,CAAC;AAEpF,mFAAmF;AACnF,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAEnC,iFAAiF;AAEjF;;;;;;GAMG;AACH,SAAS,UAAU,CAAC,GAAW,EAAE,OAAe;IAC5C,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,IAAI,OAAO,+DAA+D,OAAO,OAAO,CAAC,CAAC;IACrH,OAAO,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;AAChD,CAAC;AAED,2DAA2D;AAC3D,SAAS,cAAc,CAAC,GAAW,EAAE,OAAe;IAChD,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,IAAI,OAAO,+DAA+D,OAAO,OAAO,EAAE,GAAG,CAAC,CAAC;IAC1H,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,KAA6B,CAAC;IAClC,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,IAAI,KAAK;YAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,gFAAgF;AAChF,SAAS,gBAAgB,CAAC,GAAW,EAAE,OAAe,EAAE,SAAiB;IACrE,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,IAAI,OAAO,mBAAmB,SAAS,2BAA2B,CAAC,CAAC;IAC/F,OAAO,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;AAChD,CAAC;AAED,6DAA6D;AAC7D,SAAS,cAAc,CAAC,IAAY;IAChC,OAAO,IAAI;SACN,OAAO,CAAC,qBAAqB,EAAE,CAAC,CAAC,EAAE,GAAW,EAAE,EAAE,CAAC,aAAa,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;SACpF,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,GAAW,EAAE,EAAE,CAAC,aAAa,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;SAC1E,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;QACxB,uEAAuE;QACvE,4CAA4C;SAC3C,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;AAChC,CAAC;AAED,SAAS,aAAa,CAAC,IAAY;IAC/B,2EAA2E;IAC3E,uEAAuE;IACvE,OAAO,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AACpG,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY;IAClC,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,WAAW,GAAG,SAAS,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IACtD,OAAO,cAAc,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AACnE,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,iBAAiB,CAAC,GAAW,EAAE,QAAgB;IAC3D,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,MAAM,YAAY,GAAG,gDAAgD,CAAC;IACtE,IAAI,KAA6B,CAAC;IAElC,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC/C,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAEzB,MAAM,KAAK,GAAG,SAAS,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;QACtD,IAAI,CAAC,KAAK;YAAE,SAAS;QAErB,2EAA2E;QAC3E,0CAA0C;QAC1C,MAAM,IAAI,GACN,cAAc,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC3C,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC;YACzC,oBAAoB,CAAC,OAAO,CAAC,CAAC;QAElC,iEAAiE;QACjE,MAAM,cAAc,GAChB,UAAU,CAAC,OAAO,EAAE,aAAa,CAAC;YAClC,UAAU,CAAC,OAAO,EAAE,iBAAiB,CAAC;YACtC,UAAU,CAAC,OAAO,EAAE,SAAS,CAAC;YAC9B,UAAU,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAEnC,4EAA4E;QAC5E,MAAM,OAAO,GACT,UAAU,CAAC,OAAO,EAAE,SAAS,CAAC;YAC9B,UAAU,CAAC,OAAO,EAAE,WAAW,CAAC;YAChC,UAAU,CAAC,OAAO,EAAE,SAAS,CAAC;YAC9B,UAAU,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAEnC,sDAAsD;QACtD,MAAM,UAAU,GAAG,cAAc,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QACvD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,gBAAgB,CAAC,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;YAC3D,IAAI,IAAI;gBAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpC,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC;YACV,KAAK;YACL,IAAI;YACJ,WAAW,EAAE,SAAS,CAAC,cAAc,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,sBAAsB,CAAC;YACvE,WAAW,EAAE,aAAa,CAAC,OAAO,CAAC;YACnC,UAAU;YACV,QAAQ;SACX,CAAC,CAAC;IACP,CAAC;IAED,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED,kFAAkF;AAClF,SAAS,oBAAoB,CAAC,GAAW;IACrC,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,qEAAqE,CAAC,CAAC;IAC/F,OAAO,cAAc,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;AACpD,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,aAAa,CAAC,GAAW;IACrC,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC;IAC7B,OAAO,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;AACjE,CAAC;AAaD;;;;;;;;;;GAUG;AACH,MAAM,UAAU,WAAW,CACvB,QAAuB,EACvB,UAAkB,EAClB,GAAS,EACT,cAAuB;IAEvB,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,UAAU,GAAG,UAAU,CAAC;IACvD,MAAM,IAAI,GAAkB,EAAE,CAAC;IAC/B,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC7B,IAAI,OAAO,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;YAC/B,YAAY,EAAE,CAAC;YACf,IAAI,cAAc;gBAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvC,SAAS;QACb,CAAC;QACD,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE,GAAG,MAAM,EAAE,CAAC;YACnD,WAAW,EAAE,CAAC;YACd,SAAS;QACb,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,YAAY,EAAE,CAAC;AAC/C,CAAC;AAED,iFAAiF;AAEjF;;;;;;;;GAQG;AACH,MAAM,UAAU,qBAAqB,CAAC,OAAoB,EAAE,QAAkB;IAC1E,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;IAC1C,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC;IACtD,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAE1E,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC5C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAElC,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,YAAY,IAAI,iBAAiB,CAAC,KAAK,CAAC;QACpE,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,YAAY,IAAI,iBAAiB,CAAC,QAAQ,CAAC;QAC5E,IAAI,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,YAAY,IAAI,iBAAiB,CAAC,WAAW,CAAC;QAEhF,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YACnB,KAAK,IAAI,YAAY,CAAC;YACtB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC1B,CAAC;IACL,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;AAC9B,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,WAA0B,EAAE,cAAsB,EAAE,GAAS;IAC7F,IAAI,WAAW,KAAK,IAAI,IAAI,cAAc,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE,CAAC,GAAG,UAAU,CAAC;IAC/E,IAAI,OAAO,GAAG,cAAc;QAAE,OAAO,CAAC,CAAC;IACvC,uEAAuE;IACvE,IAAI,OAAO,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAC3B,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,OAAO,GAAG,cAAc,CAAC,CAAC;AACvD,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CAChC,QAAuB,EACvB,QAAkB,EAClB,cAAsB,EACtB,GAAS;IAET,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAqB,EAAE;QACvD,MAAM,EAAE,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,GAAG,qBAAqB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QACpF,MAAM,YAAY,GAAG,mBAAmB,CAAC,OAAO,CAAC,WAAW,EAAE,cAAc,EAAE,GAAG,CAAC,CAAC;QACnF,OAAO;YACH,OAAO;YACP,cAAc;YACd,YAAY;YACZ,UAAU,EAAE,cAAc,GAAG,EAAE,GAAG,YAAY,GAAG,CAAC;YAClD,eAAe,EAAE,OAAO;SAC3B,CAAC;IACN,CAAC,CAAC,CAAC;IAEH,4EAA4E;IAC5E,6EAA6E;IAC7E,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;AAC9D,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,cAAc,CAAC,MAA2B,EAAE,QAAkB,EAAE,YAAY,GAAG,IAAI;IAC/F,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC;IACzC,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc,GAAG,CAAC,IAAI,CAAC,CAAC,YAAY,IAAI,YAAY,CAAC,CAAC;AACtF,CAAC"}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { ActionResultSimple, RunActionParams } from "@memberjunction/actions-base";
|
|
2
|
+
import { BaseAction } from "@memberjunction/actions";
|
|
3
|
+
/** One feed to read. `Name` is what a merged result set is attributed to. */
|
|
4
|
+
export interface RSSFeedInput {
|
|
5
|
+
name: string;
|
|
6
|
+
url: string;
|
|
7
|
+
}
|
|
8
|
+
/** Per-feed outcome, reported whether the feed succeeded or not. */
|
|
9
|
+
export interface RSSFeedStatus {
|
|
10
|
+
name: string;
|
|
11
|
+
url: string;
|
|
12
|
+
success: boolean;
|
|
13
|
+
articleCount: number;
|
|
14
|
+
error: string | null;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Action that reads one or more RSS or Atom feeds and returns their articles,
|
|
18
|
+
* optionally scored and ranked against a keyword list.
|
|
19
|
+
*
|
|
20
|
+
* Feeds are fetched concurrently and a failing feed does not fail the action: its
|
|
21
|
+
* error is reported in `FeedStatuses` while the other feeds still return their
|
|
22
|
+
* articles. A feed being unreachable is the normal case at any scale — one dead
|
|
23
|
+
* URL in a list of twenty should not cost the caller the other nineteen.
|
|
24
|
+
*
|
|
25
|
+
* No credential. These are public HTTP endpoints.
|
|
26
|
+
*
|
|
27
|
+
* Scoring is a keyword-relevance weight (title 3, category 2, description 1)
|
|
28
|
+
* plus a recency decay across the time window, combined as
|
|
29
|
+
* `relevance * 10 + recency * 5` so relevance dominates and recency breaks ties.
|
|
30
|
+
* The numeric scores come back on every article rather than being bucketed into
|
|
31
|
+
* labels, because what counts as "high" depends on the caller's corpus, not on
|
|
32
|
+
* this action's batch.
|
|
33
|
+
*
|
|
34
|
+
* @example
|
|
35
|
+
* ```typescript
|
|
36
|
+
* // Just read a feed
|
|
37
|
+
* await runAction({
|
|
38
|
+
* ActionName: 'Read RSS Feed',
|
|
39
|
+
* Params: [{ Name: 'FeedURLs', Value: 'https://example.org/news/feed.xml' }]
|
|
40
|
+
* });
|
|
41
|
+
*
|
|
42
|
+
* // Several named feeds, ranked against keywords, last 7 days
|
|
43
|
+
* await runAction({
|
|
44
|
+
* ActionName: 'Read RSS Feed',
|
|
45
|
+
* Params: [
|
|
46
|
+
* { Name: 'Feeds', Value: [
|
|
47
|
+
* { name: 'Associations Now', url: 'https://associationsnow.com/feed/' },
|
|
48
|
+
* { name: 'NonprofitPRO', url: 'https://www.nonprofitpro.com/feed/' }
|
|
49
|
+
* ]},
|
|
50
|
+
* { Name: 'Keywords', Value: ['membership', 'dues', 'retention'] },
|
|
51
|
+
* { Name: 'MaxAgeDays', Value: 7 },
|
|
52
|
+
* { Name: 'MaxResults', Value: 25 }
|
|
53
|
+
* ]
|
|
54
|
+
* });
|
|
55
|
+
* ```
|
|
56
|
+
*/
|
|
57
|
+
export declare class ReadRSSFeedAction extends BaseAction {
|
|
58
|
+
private static readonly DEFAULT_MAX_AGE_DAYS;
|
|
59
|
+
private static readonly DEFAULT_MAX_RESULTS;
|
|
60
|
+
private static readonly FETCH_TIMEOUT_MS;
|
|
61
|
+
/** Bounded so one enormous feed cannot exhaust memory. */
|
|
62
|
+
private static readonly MAX_FEED_BYTES;
|
|
63
|
+
private static readonly USER_AGENT;
|
|
64
|
+
/**
|
|
65
|
+
* Executes the feed read.
|
|
66
|
+
*
|
|
67
|
+
* @param params - The action parameters containing:
|
|
68
|
+
* - Feeds: Array of `{ name, url }` (or a JSON string of one). Either this
|
|
69
|
+
* or FeedURLs is required
|
|
70
|
+
* - FeedURLs: Array or comma-separated string of URLs, named after their host
|
|
71
|
+
* - Keywords: Array or comma-separated string. Optional — with none supplied,
|
|
72
|
+
* articles come back ranked purely by recency
|
|
73
|
+
* - MaxAgeDays: Age window, default 7. News has a short shelf life
|
|
74
|
+
* - MaxResults: Cap on returned articles, default 25
|
|
75
|
+
* - IncludeUndated: Keep articles the feed gave no parseable date (default
|
|
76
|
+
* false). They cannot be age-filtered, so this is an explicit choice
|
|
77
|
+
* - RequireAllFeeds: Fail the action if any feed failed (default false)
|
|
78
|
+
*
|
|
79
|
+
* @returns Ranked articles, plus a per-feed status list
|
|
80
|
+
*/
|
|
81
|
+
protected InternalRunAction(params: RunActionParams): Promise<ActionResultSimple>;
|
|
82
|
+
/** The current time. Overridable so tests can pin the clock. */
|
|
83
|
+
protected Now(): Date;
|
|
84
|
+
/** Fetch, with a timeout and a size bound. Overridable for tests. */
|
|
85
|
+
protected FetchFeed(url: string): Promise<string>;
|
|
86
|
+
private fetchAndParse;
|
|
87
|
+
/**
|
|
88
|
+
* Resolve `Feeds` or `FeedURLs` into a validated list.
|
|
89
|
+
*
|
|
90
|
+
* A URL that is not http(s) is rejected rather than attempted: `file://` and
|
|
91
|
+
* friends would turn a feed-reading action into a local-file reader, which is
|
|
92
|
+
* not what any caller of this means.
|
|
93
|
+
*/
|
|
94
|
+
private resolveFeeds;
|
|
95
|
+
private isHttpUrl;
|
|
96
|
+
/** Fall back to the host as a feed's display name. */
|
|
97
|
+
private nameFromUrl;
|
|
98
|
+
/** Flatten a scored article for output — scores at the top level, not nested. */
|
|
99
|
+
private toOutputArticle;
|
|
100
|
+
private getParamValue;
|
|
101
|
+
private getStringArrayParam;
|
|
102
|
+
private getNumericParam;
|
|
103
|
+
private getBooleanParam;
|
|
104
|
+
private addOutputParam;
|
|
105
|
+
private createErrorResult;
|
|
106
|
+
}
|
|
107
|
+
export declare function LoadReadRSSFeedAction(): void;
|
|
108
|
+
//# sourceMappingURL=rss-feed-read.action.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rss-feed-read.action.d.ts","sourceRoot":"","sources":["../../../src/custom/web/rss-feed-read.action.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACnF,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAYrD,6EAA6E;AAC7E,MAAM,WAAW,YAAY;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;CACf;AAED,oEAAoE;AACpE,MAAM,WAAW,aAAa;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AACH,qBACa,iBAAkB,SAAQ,UAAU;IAC7C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAK;IACjD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAAM;IACjD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IACjD,0DAA0D;IAC1D,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAoB;IAC1D,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAoC;IAEtE;;;;;;;;;;;;;;;;OAgBG;cACa,iBAAiB,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,kBAAkB,CAAC;IA8EvF,gEAAgE;IAChE,SAAS,CAAC,GAAG,IAAI,IAAI;IAIrB,qEAAqE;cACrD,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;YA2CzC,aAAa;IAK3B;;;;;;OAMG;IACH,OAAO,CAAC,YAAY;IAuEpB,OAAO,CAAC,SAAS;IASjB,sDAAsD;IACtD,OAAO,CAAC,WAAW;IAQnB,iFAAiF;IACjF,OAAO,CAAC,eAAe;IAevB,OAAO,CAAC,aAAa;IAIrB,OAAO,CAAC,mBAAmB;IAO3B,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,iBAAiB;CAG5B;AAED,wBAAgB,qBAAqB,IAAI,IAAI,CAE5C"}
|