feedme 2.0.1 → 2.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/feedme.d.ts +22 -0
- package/dist/feedme.js +69 -0
- package/dist/feedme.js.map +1 -0
- package/dist/jsonfeedparser.d.ts +19 -0
- package/dist/jsonfeedparser.js +178 -0
- package/dist/jsonfeedparser.js.map +1 -0
- package/dist/parser.d.ts +18 -0
- package/dist/parser.js +3 -0
- package/dist/parser.js.map +1 -0
- package/dist/xmlfeedparser.d.ts +19 -0
- package/dist/xmlfeedparser.js +195 -0
- package/dist/xmlfeedparser.js.map +1 -0
- package/package.json +16 -7
- package/lib/feedme.js +0 -68
- package/lib/jsonfeedparser.js +0 -172
- package/lib/xmlfeedparser.js +0 -197
package/README.md
CHANGED
|
@@ -3,7 +3,8 @@
|
|
|
3
3
|
|
|
4
4
|
feedme.js is an RSS/Atom/JSON feed parser. How is this different from the other few feed parsers? It uses [sax-js](https://github.com/isaacs/sax-js) for xml parsing and [clarinet](https://github.com/dscape/clarinet) for json parsing. That means it is coded in pure Javascript and thus more deployable. I needed a parser that wouldn't require me to install external dependencies or to compile anything.
|
|
5
5
|
|
|
6
|
-
](https://www.npmjs.com/package/feedme)
|
|
7
|
+
[](https://github.com/fent/feedme.js/actions/workflows/nodejs.yml)
|
|
7
8
|
[](https://codecov.io/gh/fent/feedme.js)
|
|
8
9
|
|
|
9
10
|
|
|
@@ -11,9 +12,9 @@ feedme.js is an RSS/Atom/JSON feed parser. How is this different from the other
|
|
|
11
12
|
|
|
12
13
|
```js
|
|
13
14
|
const FeedMe = require('feedme');
|
|
14
|
-
const
|
|
15
|
+
const https = require('https');
|
|
15
16
|
|
|
16
|
-
|
|
17
|
+
https.get('https://feeds.npr.org/1001/rss.xml', (res) => {
|
|
17
18
|
if (res.statusCode != 200) {
|
|
18
19
|
console.error(new Error(`status code ${res.statusCode}`));
|
|
19
20
|
return;
|
package/dist/feedme.d.ts
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import { Writable } from 'stream';
|
|
3
|
+
export default class FeedMe extends Writable {
|
|
4
|
+
private _buffer;
|
|
5
|
+
private _parser;
|
|
6
|
+
/**
|
|
7
|
+
* Creates an instance of a parser. Parser can be JSON/XML.
|
|
8
|
+
*
|
|
9
|
+
* @param {boolean} buffer If true, feed will be buffered into memory
|
|
10
|
+
* and can be retrieved using `parser.done()` on the `end` event.
|
|
11
|
+
* @constructor
|
|
12
|
+
*/
|
|
13
|
+
constructor(buffer?: boolean);
|
|
14
|
+
_proxyEvents(): void;
|
|
15
|
+
/**
|
|
16
|
+
* @param {Buffer} data
|
|
17
|
+
*/
|
|
18
|
+
_write(data: Buffer, encoding: BufferEncoding, callback: (err?: Error | null) => void): void;
|
|
19
|
+
_final(callback: (err?: Error | null) => void): void;
|
|
20
|
+
done(): import("./parser").Feed | undefined;
|
|
21
|
+
}
|
|
22
|
+
export { Feed, FeedItem, FeedObject } from './parser';
|
package/dist/feedme.js
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const stream_1 = require("stream");
|
|
7
|
+
const xmlfeedparser_1 = __importDefault(require("./xmlfeedparser"));
|
|
8
|
+
const jsonfeedparser_1 = __importDefault(require("./jsonfeedparser"));
|
|
9
|
+
class FeedMe extends stream_1.Writable {
|
|
10
|
+
/**
|
|
11
|
+
* Creates an instance of a parser. Parser can be JSON/XML.
|
|
12
|
+
*
|
|
13
|
+
* @param {boolean} buffer If true, feed will be buffered into memory
|
|
14
|
+
* and can be retrieved using `parser.done()` on the `end` event.
|
|
15
|
+
* @constructor
|
|
16
|
+
*/
|
|
17
|
+
constructor(buffer = false) {
|
|
18
|
+
super();
|
|
19
|
+
this._buffer = buffer;
|
|
20
|
+
}
|
|
21
|
+
_proxyEvents() {
|
|
22
|
+
const parserEmit = this._parser.parser.emit;
|
|
23
|
+
this._parser.parser.emit = (event, ...args) => {
|
|
24
|
+
parserEmit.apply(this._parser.parser, [event, ...args]);
|
|
25
|
+
if (event !== 'error') {
|
|
26
|
+
return this.emit(event, ...args);
|
|
27
|
+
}
|
|
28
|
+
return true;
|
|
29
|
+
};
|
|
30
|
+
this._parser.parser.on('error', this.emit.bind(this, 'error'));
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* @param {Buffer} data
|
|
34
|
+
*/
|
|
35
|
+
_write(data, encoding, callback) {
|
|
36
|
+
const str = data.toString();
|
|
37
|
+
// First find out what type of feed this is.
|
|
38
|
+
if (!this._parser) {
|
|
39
|
+
if (/^\s*</.test(str)) {
|
|
40
|
+
this._parser = new xmlfeedparser_1.default(this._buffer);
|
|
41
|
+
this._proxyEvents();
|
|
42
|
+
}
|
|
43
|
+
else if (/^\s*[{[]/.test(str)) {
|
|
44
|
+
this._parser = new jsonfeedparser_1.default(this._buffer);
|
|
45
|
+
this.emit('type', 'json');
|
|
46
|
+
this._proxyEvents();
|
|
47
|
+
}
|
|
48
|
+
else {
|
|
49
|
+
callback(new Error('Not a correctly formatted feed'));
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
this._parser.write(data, encoding, callback);
|
|
54
|
+
}
|
|
55
|
+
_final(callback) {
|
|
56
|
+
if (this._parser) {
|
|
57
|
+
this._parser.end(callback);
|
|
58
|
+
}
|
|
59
|
+
else {
|
|
60
|
+
callback(null);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
done() {
|
|
64
|
+
return this._parser.done();
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
exports.default = FeedMe;
|
|
68
|
+
module.exports = FeedMe;
|
|
69
|
+
//# sourceMappingURL=feedme.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"feedme.js","sourceRoot":"","sources":["../src/feedme.ts"],"names":[],"mappings":";;;;;AAAA,mCAAkC;AAElC,oEAA4C;AAC5C,sEAA8C;AAG9C,MAAqB,MAAO,SAAQ,iBAAQ;IAI1C;;;;;;OAMG;IACH,YAAY,MAAM,GAAG,KAAK;QACxB,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IACxB,CAAC;IAED,YAAY;QACV,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC;QAC5C,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,KAAsB,EAAE,GAAG,IAAW,EAAW,EAAE;YAC7E,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;YACxD,IAAI,KAAK,KAAK,OAAO,EAAE;gBACrB,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;aAClC;YACD,OAAO,IAAI,CAAC;QACd,CAAC,CAAC;QACF,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;IACjE,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,IAAY,EAAE,QAAwB,EAAE,QAAsC;QACnF,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QAE5B,4CAA4C;QAC5C,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACjB,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;gBACrB,IAAI,CAAC,OAAO,GAAG,IAAI,uBAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC/C,IAAI,CAAC,YAAY,EAAE,CAAC;aAErB;iBAAM,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;gBAC/B,IAAI,CAAC,OAAO,GAAG,IAAI,wBAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAChD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBAC1B,IAAI,CAAC,YAAY,EAAE,CAAC;aAErB;iBAAM;gBACL,QAAQ,CAAC,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC,CAAC;gBACtD,OAAO;aACR;SACF;QAED,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IAED,MAAM,CAAC,QAAsC;QAC3C,IAAI,IAAI,CAAC,OAAO,EAAE;YAChB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;SAC5B;aAAM;YACL,QAAQ,CAAC,IAAI,CAAC,CAAC;SAChB;IACH,CAAC;IAED,IAAI;QACF,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IAC7B,CAAC;CACF;AAjED,yBAiEC;AAID,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import { Writable } from 'stream';
|
|
3
|
+
import { Parser, Feed } from './parser';
|
|
4
|
+
/**
|
|
5
|
+
* Parses a JSON feed.
|
|
6
|
+
*
|
|
7
|
+
* @param {boolean} buffer If true, will buffer entire object.
|
|
8
|
+
* @return {CStream}
|
|
9
|
+
*/
|
|
10
|
+
export default class JSONFeedParser extends Writable implements Parser {
|
|
11
|
+
_buffer: boolean;
|
|
12
|
+
parser: Writable;
|
|
13
|
+
private _currObj;
|
|
14
|
+
private _currKey;
|
|
15
|
+
constructor(buffer: boolean);
|
|
16
|
+
_write(chunk: Buffer, encoding: BufferEncoding, callback: (err?: Error | null) => void): void;
|
|
17
|
+
_final(callback: (err?: Error | null) => void): void;
|
|
18
|
+
done(): Feed | undefined;
|
|
19
|
+
}
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const stream_1 = require("stream");
|
|
7
|
+
const clarinet_1 = __importDefault(require("clarinet"));
|
|
8
|
+
/**
|
|
9
|
+
* Gets rid of unnecessary keys in an object
|
|
10
|
+
* and if the only keys in an object are `$t` or `$t` and `type`
|
|
11
|
+
* it makes the whole object the value of `$t`
|
|
12
|
+
* otherwise it renames `$t` to `text` for consistency.
|
|
13
|
+
*
|
|
14
|
+
* @param {Object}
|
|
15
|
+
* @returns {Object}
|
|
16
|
+
*/
|
|
17
|
+
const cleanObject = (obj) => {
|
|
18
|
+
if (obj.$t !== undefined) {
|
|
19
|
+
let keysLength = Object.keys(obj).length;
|
|
20
|
+
if (keysLength === 1 || (keysLength === 2 && obj.type !== undefined)) {
|
|
21
|
+
return obj.$t;
|
|
22
|
+
}
|
|
23
|
+
else {
|
|
24
|
+
obj.text = obj.$t;
|
|
25
|
+
delete obj.$t;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
return obj;
|
|
29
|
+
};
|
|
30
|
+
/**
|
|
31
|
+
* Parses a JSON feed.
|
|
32
|
+
*
|
|
33
|
+
* @param {boolean} buffer If true, will buffer entire object.
|
|
34
|
+
* @return {CStream}
|
|
35
|
+
*/
|
|
36
|
+
class JSONFeedParser extends stream_1.Writable {
|
|
37
|
+
constructor(buffer) {
|
|
38
|
+
super();
|
|
39
|
+
this._buffer = buffer;
|
|
40
|
+
const parser = this.parser = clarinet_1.default.createStream();
|
|
41
|
+
const stack = [];
|
|
42
|
+
this._currObj = {};
|
|
43
|
+
this._currKey = 'feed';
|
|
44
|
+
let inArray = false;
|
|
45
|
+
let feedJustFound = false;
|
|
46
|
+
// Look feed object in case this is a json encoded atom feed.
|
|
47
|
+
// Place these underlying keys onto the root object.
|
|
48
|
+
const findfeed = (key) => {
|
|
49
|
+
if (key === 'feed') {
|
|
50
|
+
feedJustFound = true;
|
|
51
|
+
parser.removeListener('openobject', findfeed);
|
|
52
|
+
parser.removeListener('key', findfeed);
|
|
53
|
+
}
|
|
54
|
+
};
|
|
55
|
+
const onvalue = (value) => {
|
|
56
|
+
feedJustFound = false;
|
|
57
|
+
this._currObj[this._currKey] = value;
|
|
58
|
+
if (stack.length === 1) {
|
|
59
|
+
parser.emit(this._currKey, value);
|
|
60
|
+
if (!buffer) {
|
|
61
|
+
delete this._currObj[this._currKey];
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
if (inArray) {
|
|
65
|
+
this._currKey++;
|
|
66
|
+
}
|
|
67
|
+
};
|
|
68
|
+
const onopenobject = (key) => {
|
|
69
|
+
if (feedJustFound) {
|
|
70
|
+
feedJustFound = false;
|
|
71
|
+
this._currKey = key;
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
let obj = this._currObj[this._currKey] = {};
|
|
75
|
+
stack.push({
|
|
76
|
+
obj: this._currObj,
|
|
77
|
+
key: this._currKey,
|
|
78
|
+
arr: inArray,
|
|
79
|
+
});
|
|
80
|
+
this._currObj = obj;
|
|
81
|
+
this._currKey = key;
|
|
82
|
+
inArray = false;
|
|
83
|
+
};
|
|
84
|
+
const onkey = (key) => { this._currKey = key; };
|
|
85
|
+
const oncloseobject = () => {
|
|
86
|
+
let parent = stack.pop();
|
|
87
|
+
if (!parent) {
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
this._currObj = parent.obj;
|
|
91
|
+
this._currKey = parent.key;
|
|
92
|
+
inArray = parent.arr;
|
|
93
|
+
// Clean object.
|
|
94
|
+
const currObj = this._currObj;
|
|
95
|
+
currObj[this._currKey] = cleanObject(currObj[this._currKey]);
|
|
96
|
+
// Emit key in feed if curr is parent.
|
|
97
|
+
if (stack.length === 1) {
|
|
98
|
+
parser.emit(`${this._currKey}`, currObj[this._currKey]);
|
|
99
|
+
if (!buffer) {
|
|
100
|
+
delete currObj[this._currKey];
|
|
101
|
+
}
|
|
102
|
+
// Or parent is array.
|
|
103
|
+
}
|
|
104
|
+
else if (inArray) {
|
|
105
|
+
const currArr = currObj;
|
|
106
|
+
if (stack.length === 2) {
|
|
107
|
+
let key = stack[1].key;
|
|
108
|
+
let event = key === 'entry' || key === 'items' ?
|
|
109
|
+
'item' : stack[1].key;
|
|
110
|
+
let data = currArr[this._currKey];
|
|
111
|
+
parser.emit(event, data);
|
|
112
|
+
if (!buffer) {
|
|
113
|
+
currArr.splice(this._currKey, 1);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
if (stack.length > 2 || buffer) {
|
|
117
|
+
this._currKey++;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
};
|
|
121
|
+
const onopenarray = () => {
|
|
122
|
+
feedJustFound = false;
|
|
123
|
+
let obj = this._currObj[this._currKey] = [];
|
|
124
|
+
stack.push({
|
|
125
|
+
obj: this._currObj,
|
|
126
|
+
key: this._currKey,
|
|
127
|
+
arr: inArray,
|
|
128
|
+
});
|
|
129
|
+
this._currObj = obj;
|
|
130
|
+
this._currKey = 0;
|
|
131
|
+
inArray = true;
|
|
132
|
+
};
|
|
133
|
+
const onclosearray = () => {
|
|
134
|
+
let parent = stack.pop();
|
|
135
|
+
this._currObj = parent.obj;
|
|
136
|
+
this._currKey = parent.key;
|
|
137
|
+
inArray = parent.arr;
|
|
138
|
+
if (stack.length === 1) {
|
|
139
|
+
if (!buffer) {
|
|
140
|
+
delete this._currObj[this._currKey];
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
else if (inArray) {
|
|
144
|
+
this._currKey++;
|
|
145
|
+
}
|
|
146
|
+
};
|
|
147
|
+
parser.on('openobject', findfeed);
|
|
148
|
+
parser.on('key', findfeed);
|
|
149
|
+
parser.on('value', onvalue);
|
|
150
|
+
parser.on('openobject', onopenobject);
|
|
151
|
+
parser.on('key', onkey);
|
|
152
|
+
parser.on('closeobject', oncloseobject);
|
|
153
|
+
parser.on('openarray', onopenarray);
|
|
154
|
+
parser.on('closearray', onclosearray);
|
|
155
|
+
}
|
|
156
|
+
_write(chunk, encoding, callback) {
|
|
157
|
+
this.parser.write(chunk, encoding);
|
|
158
|
+
callback(null);
|
|
159
|
+
}
|
|
160
|
+
_final(callback) {
|
|
161
|
+
this.parser.end();
|
|
162
|
+
callback(null);
|
|
163
|
+
}
|
|
164
|
+
done() {
|
|
165
|
+
if (!this._buffer) {
|
|
166
|
+
return;
|
|
167
|
+
}
|
|
168
|
+
let root = this._currObj[this._currKey];
|
|
169
|
+
root.type = 'json';
|
|
170
|
+
if (Array.isArray(root.entry)) {
|
|
171
|
+
root.items = root.entry;
|
|
172
|
+
}
|
|
173
|
+
delete root.entry;
|
|
174
|
+
return root;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
exports.default = JSONFeedParser;
|
|
178
|
+
//# sourceMappingURL=jsonfeedparser.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"jsonfeedparser.js","sourceRoot":"","sources":["../src/jsonfeedparser.ts"],"names":[],"mappings":";;;;;AAAA,mCAAkC;AAClC,wDAA+C;AAgB/C;;;;;;;;GAQG;AACH,MAAM,WAAW,GAAG,CAAC,GAAe,EAAE,EAAE;IACtC,IAAI,GAAG,CAAC,EAAE,KAAK,SAAS,EAAE;QACxB,IAAI,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;QACzC,IAAI,UAAU,KAAK,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,EAAE;YACpE,OAAO,GAAG,CAAC,EAAE,CAAC;SACf;aAAM;YACL,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;YAClB,OAAO,GAAG,CAAC,EAAE,CAAC;SACf;KACF;IAED,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAGF;;;;;GAKG;AACH,MAAqB,cAAe,SAAQ,iBAAQ;IAMlD,YAAY,MAAe;QACzB,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,GAAG,kBAAQ,CAAC,YAAY,EAAyB,CAAC;QAM5E,MAAM,KAAK,GAAgB,EAAE,CAAC;QAC9B,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;QACnB,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC;QACvB,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,IAAI,aAAa,GAAG,KAAK,CAAC;QAE1B,6DAA6D;QAC7D,oDAAoD;QACpD,MAAM,QAAQ,GAAG,CAAC,GAAW,EAAE,EAAE;YAC/B,IAAI,GAAG,KAAK,MAAM,EAAE;gBAClB,aAAa,GAAG,IAAI,CAAC;gBACrB,MAAM,CAAC,cAAc,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;gBAC9C,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;aACxC;QACH,CAAC,CAAC;QAEF,MAAM,OAAO,GAAG,CAAC,KAAgB,EAAE,EAAE;YACnC,aAAa,GAAG,KAAK,CAAC;YACrB,IAAI,CAAC,QAAuB,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,KAAK,CAAC;YACrD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;gBACtB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,QAAkB,EAAE,KAAK,CAAC,CAAC;gBAC5C,IAAI,CAAC,MAAM,EAAE;oBAAE,OAAQ,IAAI,CAAC,QAAuB,CAAC,IAAI,CAAC,QAAkB,CAAC,CAAC;iBAAE;aAChF;YACD,IAAI,OAAO,EAAE;gBACV,IAAI,CAAC,QAAmB,EAAE,CAAC;aAC7B;QACH,CAAC,CAAC;QAEF,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,EAAE;YACnC,IAAI,aAAa,EAAE;gBACjB,aAAa,GAAG,KAAK,CAAC;gBACtB,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC;gBACpB,OAAO;aACR;YACD,IAAI,GAAG,GAAI,IAAI,CAAC,QAAuB,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC;YAC5D,KAAK,CAAC,IAAI,CAAC;gBACT,GAAG,EAAE,IAAI,CAAC,QAAQ;gBAClB,GAAG,EAAE,IAAI,CAAC,QAAQ;gBAClB,GAAG,EAAE,OAAO;aACb,CAAC,CAAC;YACH,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC;YACpB,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC;YACpB,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC,CAAC;QAEF,MAAM,KAAK,GAAG,CAAC,GAAW,EAAE,EAAE,GAAG,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QAExD,MAAM,aAAa,GAAG,GAAG,EAAE;YACzB,IAAI,MAAM,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC;YACzB,IAAI,CAAC,MAAM,EAAE;gBAAE,OAAO;aAAE;YACxB,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC;YAC3B,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC;YAC3B,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC;YAErB,gBAAgB;YAChB,MAAM,OAAO,GAAG,IAAI,CAAC,QAAsB,CAAC;YAC5C,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAe,CAAC,CAAC;YAE3E,sCAAsC;YACtC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;gBACtB,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,QAAQ,EAAE,EAAE,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;gBACxD,IAAI,CAAC,MAAM,EAAE;oBAAE,OAAO,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;iBAAE;gBAE/C,sBAAsB;aACvB;iBAAM,IAAI,OAAO,EAAE;gBAClB,MAAM,OAAO,GAAG,OAA+B,CAAC;gBAChD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;oBACtB,IAAI,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;oBACvB,IAAI,KAAK,GAAG,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,OAAO,CAAC,CAAC;wBAC9C,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAa,CAAC;oBAClC,IAAI,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,QAAkB,CAAC,CAAC;oBAC5C,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;oBACzB,IAAI,CAAC,MAAM,EAAE;wBAAE,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,QAAkB,EAAE,CAAC,CAAC,CAAC;qBAAE;iBAC7D;gBAED,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,EAAE;oBAAG,IAAI,CAAC,QAAmB,EAAE,CAAC;iBAAE;aACjE;QACH,CAAC,CAAC;QAEF,MAAM,WAAW,GAAG,GAAG,EAAE;YACvB,aAAa,GAAG,KAAK,CAAC;YACtB,IAAI,GAAG,GAAa,IAAI,CAAC,QAAuB,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC;YACrE,KAAK,CAAC,IAAI,CAAC;gBACT,GAAG,EAAE,IAAI,CAAC,QAAQ;gBAClB,GAAG,EAAE,IAAI,CAAC,QAAQ;gBAClB,GAAG,EAAE,OAAO;aACb,CAAC,CAAC;YACH,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC;YACpB,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YAClB,OAAO,GAAG,IAAI,CAAC;QACjB,CAAC,CAAC;QAEF,MAAM,YAAY,GAAG,GAAG,EAAE;YACxB,IAAI,MAAM,GAAG,KAAK,CAAC,GAAG,EAAe,CAAC;YACtC,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC;YAC3B,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC;YAC3B,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC;YAErB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;gBACtB,IAAI,CAAC,MAAM,EAAE;oBAAE,OAAQ,IAAI,CAAC,QAAuB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;iBAAE;aACtE;iBAAM,IAAI,OAAO,EAAE;gBACjB,IAAI,CAAC,QAAmB,EAAE,CAAC;aAC7B;QACH,CAAC,CAAC;QAEF,MAAM,CAAC,EAAE,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;QAClC,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC3B,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC5B,MAAM,CAAC,EAAE,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;QACtC,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACxB,MAAM,CAAC,EAAE,CAAC,aAAa,EAAE,aAAa,CAAC,CAAC;QACxC,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;QACpC,MAAM,CAAC,EAAE,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,CAAC,KAAa,EAAE,QAAwB,EAAE,QAAsC;QACpF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QACnC,QAAQ,CAAC,IAAI,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,CAAC,QAAsC;QAC3C,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;QAClB,QAAQ,CAAC,IAAI,CAAC,CAAC;IACjB,CAAC;IAED,IAAI;QACF,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YAAE,OAAO;SAAE;QAC9B,IAAI,IAAI,GAAI,IAAI,CAAC,QAAuB,CAAC,IAAI,CAAC,QAAQ,CAAa,CAAC;QACpE,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC;QACnB,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;YAC7B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;SACzB;QACD,OAAO,IAAI,CAAC,KAAK,CAAC;QAClB,OAAO,IAAY,CAAC;IACtB,CAAC;CACF;AAtJD,iCAsJC"}
|
package/dist/parser.d.ts
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import { Writable } from 'stream';
|
|
3
|
+
export declare type FeedObject = string | {
|
|
4
|
+
[key: string]: string | boolean | null | FeedObject | FeedObject[];
|
|
5
|
+
};
|
|
6
|
+
export declare type FeedItem = {
|
|
7
|
+
[key: string]: FeedObject;
|
|
8
|
+
};
|
|
9
|
+
export declare type Feed = {
|
|
10
|
+
type: string;
|
|
11
|
+
items: FeedItem[];
|
|
12
|
+
[key: string]: FeedObject | FeedObject[];
|
|
13
|
+
};
|
|
14
|
+
export interface Parser extends Writable {
|
|
15
|
+
_buffer: boolean;
|
|
16
|
+
parser: Writable;
|
|
17
|
+
done(): Feed | undefined;
|
|
18
|
+
}
|
package/dist/parser.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parser.js","sourceRoot":"","sources":["../src/parser.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import { Writable } from 'stream';
|
|
3
|
+
import { Parser, Feed } from './parser';
|
|
4
|
+
/**
|
|
5
|
+
* Parses an RSS/Atom feed.
|
|
6
|
+
*
|
|
7
|
+
* @param {boolean} buffer If true, will buffer entire object.
|
|
8
|
+
* @return {sax.Stream}
|
|
9
|
+
*/
|
|
10
|
+
export default class XMLFeedParser extends Writable implements Parser {
|
|
11
|
+
_buffer: boolean;
|
|
12
|
+
parser: Writable;
|
|
13
|
+
private _root;
|
|
14
|
+
private _obj;
|
|
15
|
+
constructor(buffer: boolean);
|
|
16
|
+
_write(chunk: Buffer, encoding: BufferEncoding, callback: (err?: Error | null) => void): void;
|
|
17
|
+
_final(callback: (err?: Error | null) => void): void;
|
|
18
|
+
done(): Feed | undefined;
|
|
19
|
+
}
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const stream_1 = require("stream");
|
|
7
|
+
const sax_1 = __importDefault(require("sax"));
|
|
8
|
+
/**
|
|
9
|
+
* Convert simple XML elements into text values. Elements with no content
|
|
10
|
+
* become an empty string, including typed empty text elements such as
|
|
11
|
+
* `<summary type="text"></summary>`.
|
|
12
|
+
*
|
|
13
|
+
* @param {Object} obj
|
|
14
|
+
* @returns {Object}
|
|
15
|
+
*/
|
|
16
|
+
const cleanObj = (obj) => {
|
|
17
|
+
const keys = Object.keys(obj).length;
|
|
18
|
+
const hasOnlyTypeAttribute = keys === 1 && obj.type != null;
|
|
19
|
+
if (obj.text != null) {
|
|
20
|
+
// If this tag has text, trim it.
|
|
21
|
+
obj.text = trimIndent(obj.text);
|
|
22
|
+
if (keys === 1 || (keys === 2 && obj.type != null)) {
|
|
23
|
+
return obj.text;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
else if (keys === 0 || hasOnlyTypeAttribute) {
|
|
27
|
+
return '';
|
|
28
|
+
}
|
|
29
|
+
return obj;
|
|
30
|
+
};
|
|
31
|
+
/**
|
|
32
|
+
* Trims the indent from text that sax-js emits. Some xml feeds will
|
|
33
|
+
* add indent to multiline text. Example:
|
|
34
|
+
*
|
|
35
|
+
* <item>
|
|
36
|
+
* The big brown fox
|
|
37
|
+
* jumped over the lazy dog
|
|
38
|
+
* </item>
|
|
39
|
+
*
|
|
40
|
+
* @param {string} str
|
|
41
|
+
* @returns {string}
|
|
42
|
+
*/
|
|
43
|
+
const trimIndent = (str) => {
|
|
44
|
+
let split = str.split('\n'), rs;
|
|
45
|
+
// The last line should be indented and only contain whitespace.
|
|
46
|
+
if (split.length > 1 &&
|
|
47
|
+
(rs = /^((?: {3})+|(?: {2})+|(?: {4})+|(?:\t)+)$/m.exec(split[split.length - 1]))) {
|
|
48
|
+
// The very first line should be empty.
|
|
49
|
+
if (split.shift() !== '') {
|
|
50
|
+
return str;
|
|
51
|
+
}
|
|
52
|
+
// Remove the last line, as it's only whitespace.
|
|
53
|
+
split.pop();
|
|
54
|
+
let wholeindent = rs[0] + rs[1];
|
|
55
|
+
// Remove indent from beginning of every line.
|
|
56
|
+
for (let i = 0, l = split.length; i < l; i++) {
|
|
57
|
+
let s = split[i];
|
|
58
|
+
if (s.indexOf(wholeindent) !== 0) {
|
|
59
|
+
return str;
|
|
60
|
+
}
|
|
61
|
+
split[i] = s.substr(wholeindent.length);
|
|
62
|
+
}
|
|
63
|
+
str = split.join('\n');
|
|
64
|
+
}
|
|
65
|
+
return str;
|
|
66
|
+
};
|
|
67
|
+
/**
|
|
68
|
+
* Parses an RSS/Atom feed.
|
|
69
|
+
*
|
|
70
|
+
* @param {boolean} buffer If true, will buffer entire object.
|
|
71
|
+
* @return {sax.Stream}
|
|
72
|
+
*/
|
|
73
|
+
class XMLFeedParser extends stream_1.Writable {
|
|
74
|
+
constructor(buffer) {
|
|
75
|
+
super();
|
|
76
|
+
this._buffer = buffer;
|
|
77
|
+
const parser = this.parser = sax_1.default.createStream(false, { lowercase: true });
|
|
78
|
+
const stack = [];
|
|
79
|
+
this._root = this._obj = {};
|
|
80
|
+
// First start listening for the root tag.
|
|
81
|
+
const openf1 = (node) => {
|
|
82
|
+
if (node.name === 'channel' || node.name === 'feed') {
|
|
83
|
+
if (node.name === 'feed') {
|
|
84
|
+
let type = 'atom';
|
|
85
|
+
this._root.type = type;
|
|
86
|
+
parser.emit('type', type);
|
|
87
|
+
}
|
|
88
|
+
parser.removeListener('opentag', openf1);
|
|
89
|
+
parser.on('text', ontext);
|
|
90
|
+
parser.on('cdata', ontext);
|
|
91
|
+
parser.on('opentag', onopentag);
|
|
92
|
+
parser.on('closetag', onclosetag);
|
|
93
|
+
}
|
|
94
|
+
else if (node.name === 'rss' || node.name === 'rdf:rdf') {
|
|
95
|
+
let type = 'rss ' + (node.attributes.version || '1.0');
|
|
96
|
+
this._root.type = type;
|
|
97
|
+
parser.emit('type', type);
|
|
98
|
+
}
|
|
99
|
+
else {
|
|
100
|
+
parser.removeListener('opentag', openf1);
|
|
101
|
+
parser.emit('error', new Error('Feed type not recognized'));
|
|
102
|
+
// parser.closed = true;
|
|
103
|
+
}
|
|
104
|
+
};
|
|
105
|
+
parser.on('opentag', openf1);
|
|
106
|
+
const ontext = (text) => {
|
|
107
|
+
// Make sure text events are text and not just whitespace.
|
|
108
|
+
const rs = /\n(( {3})+|( {2})+|( {4})+|(\t)+)$/m.exec(text);
|
|
109
|
+
if (rs !== null && rs.index === 0 && rs[0] === rs.input) {
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
const obj = this._obj;
|
|
113
|
+
if (obj.text == null) {
|
|
114
|
+
obj.text = '';
|
|
115
|
+
}
|
|
116
|
+
obj.text += text;
|
|
117
|
+
};
|
|
118
|
+
// After the root is found, start storing the rest of the document.
|
|
119
|
+
const onopentag = (node) => {
|
|
120
|
+
const key = node.name;
|
|
121
|
+
let i;
|
|
122
|
+
if (this._obj[key]) {
|
|
123
|
+
if (!Array.isArray(this._obj[key])) {
|
|
124
|
+
this._obj[key] = [this._obj[key], node.attributes];
|
|
125
|
+
i = 1;
|
|
126
|
+
}
|
|
127
|
+
else {
|
|
128
|
+
i = this._obj[key].push(node.attributes) - 1;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
else {
|
|
132
|
+
this._obj[key] = node.attributes;
|
|
133
|
+
}
|
|
134
|
+
stack.push({ obj: this._obj, key, i });
|
|
135
|
+
this._obj = node.attributes;
|
|
136
|
+
};
|
|
137
|
+
const onclosetag = () => {
|
|
138
|
+
const parent = stack.pop();
|
|
139
|
+
if (!parent) {
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
this._obj = parent.obj;
|
|
143
|
+
const key = parent.key;
|
|
144
|
+
const data = parent.i ?
|
|
145
|
+
this._obj[key][parent.i] = cleanObj(this._obj[key][parent.i]) :
|
|
146
|
+
this._obj[key] = cleanObj(this._obj[key]);
|
|
147
|
+
if (!stack.length) {
|
|
148
|
+
let skey = key === 'entry' || key === 'items' ? 'item' : key;
|
|
149
|
+
// Some feeds will contain a summary of items at the top.
|
|
150
|
+
// Ignore this.
|
|
151
|
+
if (key !== 'items' || Array.isArray(data)) {
|
|
152
|
+
parser.emit(skey, data);
|
|
153
|
+
if (!this._buffer) {
|
|
154
|
+
delete this._obj[key];
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
_write(chunk, encoding, callback) {
|
|
161
|
+
this.parser.write(chunk, encoding);
|
|
162
|
+
callback(null);
|
|
163
|
+
}
|
|
164
|
+
_final(callback) {
|
|
165
|
+
this.parser.end();
|
|
166
|
+
callback(null);
|
|
167
|
+
}
|
|
168
|
+
// Called when done parsing the document.
|
|
169
|
+
// Returns entire document in object form.
|
|
170
|
+
done() {
|
|
171
|
+
if (!this._buffer) {
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
let items;
|
|
175
|
+
if (this._root.item != null) {
|
|
176
|
+
items = this._root.item;
|
|
177
|
+
delete this._root.item;
|
|
178
|
+
}
|
|
179
|
+
else if (this._root.entry != null) {
|
|
180
|
+
items = this._root.entry;
|
|
181
|
+
delete this._root.entry;
|
|
182
|
+
}
|
|
183
|
+
else {
|
|
184
|
+
items = [];
|
|
185
|
+
}
|
|
186
|
+
if (!Array.isArray(items)) {
|
|
187
|
+
items = [items];
|
|
188
|
+
}
|
|
189
|
+
this._root.items = items;
|
|
190
|
+
delete this._root.text;
|
|
191
|
+
return this._root;
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
exports.default = XMLFeedParser;
|
|
195
|
+
//# sourceMappingURL=xmlfeedparser.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"xmlfeedparser.js","sourceRoot":"","sources":["../src/xmlfeedparser.ts"],"names":[],"mappings":";;;;;AAAA,mCAAkC;AAClC,8CAA4E;AAuB5E;;;;;;;GAOG;AACH,MAAM,QAAQ,GAAG,CAAC,GAAiB,EAAE,EAAE;IACrC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,MAAM,oBAAoB,GAAG,IAAI,KAAK,CAAC,IAAI,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC;IAC5D,IAAI,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE;QACpB,iCAAiC;QACjC,GAAG,CAAC,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAEhC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,IAAI,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE;YAClD,OAAO,GAAG,CAAC,IAAI,CAAC;SACjB;KACF;SAAM,IAAI,IAAI,KAAK,CAAC,IAAI,oBAAoB,EAAE;QAC7C,OAAO,EAAE,CAAC;KACX;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAGF;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,EAAE;IACjC,IAAI,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;IAEhC,gEAAgE;IAChE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QACjB,CAAC,EAAE,GAAG,4CAA4C,CAAC,IAAI,CACrD,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE;QAE9B,uCAAuC;QACvC,IAAI,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;YACxB,OAAO,GAAG,CAAC;SACZ;QAED,iDAAiD;QACjD,KAAK,CAAC,GAAG,EAAE,CAAC;QAEZ,IAAI,WAAW,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QAEhC,8CAA8C;QAC9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE;YAC5C,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACjB,IAAI,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE;gBAChC,OAAO,GAAG,CAAC;aACZ;YACD,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;SACzC;QAED,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;KACxB;IAED,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAGF;;;;;GAKG;AACH,MAAqB,aAAc,SAAQ,iBAAQ;IAMjD,YAAY,MAAe;QACzB,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,GAAG,aAAG,CAAC,YAAY,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1E,MAAM,KAAK,GAIL,EAAE,CAAC;QACT,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC;QAE5B,0CAA0C;QAC1C,MAAM,MAAM,GAAG,CAAC,IAAwB,EAAE,EAAE;YAC1C,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE;gBACnD,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE;oBACxB,IAAI,IAAI,GAAG,MAAM,CAAC;oBAClB,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC;oBACvB,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;iBAC3B;gBACD,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;gBACzC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBAC1B,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;gBAC3B,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;gBAChC,MAAM,CAAC,EAAE,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;aAEnC;iBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE;gBACzD,IAAI,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,IAAI,KAAK,CAAC,CAAC;gBACvD,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC;gBACvB,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;aAC3B;iBAAM;gBACL,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;gBACzC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC,CAAC;gBAC5D,wBAAwB;aACzB;QACH,CAAC,CAAC;QACF,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QAE7B,MAAM,MAAM,GAAG,CAAC,IAAY,EAAE,EAAE;YAC9B,0DAA0D;YAC1D,MAAM,EAAE,GAAG,qCAAqC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5D,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,CAAC,KAAK,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE;gBAAE,OAAO;aAAE;YACpE,MAAM,GAAG,GAAG,IAAI,CAAC,IAAoB,CAAC;YACtC,IAAI,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE;gBAAE,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC;aAAE;YACxC,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC;QACnB,CAAC,CAAC;QAEF,mEAAmE;QACnE,MAAM,SAAS,GAAG,CAAC,IAAwB,EAAE,EAAE;YAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC;YACtB,IAAI,CAAC,CAAC;YAEN,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;gBAClB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE;oBAClC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,UAAU,CAAgB,CAAC;oBAClE,CAAC,GAAG,CAAC,CAAC;iBACP;qBAAM;oBACL,CAAC,GAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;iBAC/D;aAEF;iBAAM;gBACL,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC;aAClC;YAED,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YACvC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,UAAoB,CAAC;QACxC,CAAC,CAAC;QAEF,MAAM,UAAU,GAAG,GAAG,EAAE;YACtB,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC;YAC3B,IAAI,CAAC,MAAM,EAAE;gBAAE,OAAO;aAAE;YAExB,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,GAAa,CAAC;YACjC,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;YACvB,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;gBACpB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAiB,CAAC,CAAC;YAE5D,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE;gBACjB,IAAI,IAAI,GAAG,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC;gBAC7D,yDAAyD;gBACzD,eAAe;gBACf,IAAI,GAAG,KAAK,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;oBAC1C,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;oBACxB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;wBAAE,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;qBAAE;iBAC9C;aACF;QACH,CAAC,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,KAAa,EAAE,QAAwB,EAAE,QAAsC;QACpF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QACnC,QAAQ,CAAC,IAAI,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,CAAC,QAAsC;QAC3C,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;QAClB,QAAQ,CAAC,IAAI,CAAC,CAAC;IACjB,CAAC;IAED,yCAAyC;IACzC,0CAA0C;IAC1C,IAAI;QACF,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YAAE,OAAO;SAAE;QAC9B,IAAI,KAA8B,CAAC;QACnC,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,EAAE;YAC3B,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;YACxB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;SACxB;aAAM,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,IAAI,EAAE;YACnC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;YACzB,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;SACzB;aAAM;YACL,KAAK,GAAG,EAAE,CAAC;SACZ;QACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAC;YACxB,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC;SACjB;QACD,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC;QACzB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;QACvB,OAAO,IAAI,CAAC,KAAa,CAAC;IAC5B,CAAC;CACF;AA9HD,gCA8HC"}
|
package/package.json
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
"json",
|
|
9
9
|
"parser"
|
|
10
10
|
],
|
|
11
|
-
"version": "2.0.
|
|
11
|
+
"version": "2.0.3",
|
|
12
12
|
"repository": {
|
|
13
13
|
"type": "git",
|
|
14
14
|
"url": "git://github.com/fent/feedme.js.git"
|
|
@@ -17,23 +17,32 @@
|
|
|
17
17
|
"contributors": [
|
|
18
18
|
"Lucas Stadler"
|
|
19
19
|
],
|
|
20
|
-
"main": "./
|
|
20
|
+
"main": "./dist/feedme.js",
|
|
21
21
|
"files": [
|
|
22
|
-
"
|
|
22
|
+
"dist"
|
|
23
23
|
],
|
|
24
24
|
"scripts": {
|
|
25
|
-
"
|
|
25
|
+
"prepare": "tsc -p tsconfig.build.json",
|
|
26
|
+
"build": "tsc -p tsconfig.build.json",
|
|
27
|
+
"test": "nyc --extension .ts --reporter=lcov --reporter=text-summary npm run test:unit",
|
|
28
|
+
"test:unit": "mocha --require ts-node/register test/*-test.ts"
|
|
26
29
|
},
|
|
27
30
|
"dependencies": {
|
|
28
31
|
"clarinet": "^0.12.4",
|
|
29
32
|
"sax": "^1.2.4"
|
|
30
33
|
},
|
|
31
34
|
"devDependencies": {
|
|
32
|
-
"
|
|
33
|
-
"
|
|
35
|
+
"@types/clarinet": "^0.12.0",
|
|
36
|
+
"@types/mocha": "^8.0.4",
|
|
37
|
+
"@types/node": "^14.14.10",
|
|
38
|
+
"@types/sax": "^1.2.1",
|
|
39
|
+
"mocha": "^12.0.3",
|
|
40
|
+
"nyc": "^18.0.0",
|
|
41
|
+
"ts-node": "^9.0.0",
|
|
42
|
+
"typescript": "^4.1.2"
|
|
34
43
|
},
|
|
35
44
|
"engines": {
|
|
36
|
-
"node": ">=
|
|
45
|
+
"node": ">=10"
|
|
37
46
|
},
|
|
38
47
|
"license": "MIT"
|
|
39
48
|
}
|
package/lib/feedme.js
DELETED
|
@@ -1,68 +0,0 @@
|
|
|
1
|
-
const { Writable } = require('stream');
|
|
2
|
-
const XMLFeedParser = require('./xmlfeedparser');
|
|
3
|
-
const JSONFeedParser = require('./jsonfeedparser');
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
module.exports = class FeedMe extends Writable {
|
|
7
|
-
/**
|
|
8
|
-
* Creates an instance of a parser. Parser can be JSON/XML.
|
|
9
|
-
*
|
|
10
|
-
* @param {boolean} buffer If true, feed will be buffered into memory
|
|
11
|
-
* and can be retrieved using `parser.done()` on the `end` event.
|
|
12
|
-
* @constructor
|
|
13
|
-
*/
|
|
14
|
-
constructor(buffer) {
|
|
15
|
-
super();
|
|
16
|
-
this._buffer = buffer;
|
|
17
|
-
this._parser = null;
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
_proxyEvents() {
|
|
21
|
-
const parserEmit = this._parser.parser.emit;
|
|
22
|
-
this._parser.parser.emit = (event, value) => {
|
|
23
|
-
parserEmit.call(this._parser.parser, event, value);
|
|
24
|
-
if (event !== 'error') {
|
|
25
|
-
this.emit(event, value);
|
|
26
|
-
}
|
|
27
|
-
};
|
|
28
|
-
this._parser.parser.on('error', this.emit.bind(this, 'error'));
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
/**
|
|
32
|
-
* @param {Buffer} data
|
|
33
|
-
*/
|
|
34
|
-
_write(data, encoding, callback) {
|
|
35
|
-
const str = data.toString();
|
|
36
|
-
|
|
37
|
-
// First find out what type of feed this is.
|
|
38
|
-
if (!this._parser) {
|
|
39
|
-
if (/^\s*</.test(str)) {
|
|
40
|
-
this._parser = new XMLFeedParser(this._buffer);
|
|
41
|
-
this._proxyEvents();
|
|
42
|
-
|
|
43
|
-
} else if (/^\s*[{[]/.test(str)) {
|
|
44
|
-
this._parser = new JSONFeedParser(this._buffer);
|
|
45
|
-
this.emit('type', 'json');
|
|
46
|
-
this._proxyEvents();
|
|
47
|
-
|
|
48
|
-
} else {
|
|
49
|
-
callback(new Error('Not a correctly formatted feed'));
|
|
50
|
-
return;
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
this._parser.write(data, encoding, callback);
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
_final(callback) {
|
|
58
|
-
if (this._parser) {
|
|
59
|
-
this._parser.end(callback);
|
|
60
|
-
} else {
|
|
61
|
-
callback(null);
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
done() {
|
|
66
|
-
return this._parser.done();
|
|
67
|
-
}
|
|
68
|
-
};
|
package/lib/jsonfeedparser.js
DELETED
|
@@ -1,172 +0,0 @@
|
|
|
1
|
-
const { Writable } = require('stream');
|
|
2
|
-
const clarinet = require('clarinet');
|
|
3
|
-
|
|
4
|
-
/**
|
|
5
|
-
* Gets rid of unnecessary keys in an object
|
|
6
|
-
* and if the only keys in an object are `$t` or `$t` and `type`
|
|
7
|
-
* it makes the whole object the value of `$t`
|
|
8
|
-
* otherwise it renames `$t` to `text` for consistency.
|
|
9
|
-
*
|
|
10
|
-
* @param {Object}
|
|
11
|
-
* @returns {Object}
|
|
12
|
-
*/
|
|
13
|
-
const cleanObject = (obj) => {
|
|
14
|
-
if (obj.$t !== undefined) {
|
|
15
|
-
let keysLength = Object.keys(obj).length;
|
|
16
|
-
if (keysLength === 1 || (keysLength === 2 && obj.type !== undefined)) {
|
|
17
|
-
return obj.$t;
|
|
18
|
-
} else {
|
|
19
|
-
obj.text = obj.$t;
|
|
20
|
-
delete obj.$t;
|
|
21
|
-
}
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
return obj;
|
|
25
|
-
};
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
/**
|
|
29
|
-
* Parses a JSON feed.
|
|
30
|
-
*
|
|
31
|
-
* @param {boolean} buffer If true, will buffer entire object.
|
|
32
|
-
* @return {clarinet.Stream}
|
|
33
|
-
*/
|
|
34
|
-
module.exports = class JSONFeedParser extends Writable {
|
|
35
|
-
constructor(buffer) {
|
|
36
|
-
super();
|
|
37
|
-
this._buffer = buffer;
|
|
38
|
-
const parser = this.parser = clarinet.createStream();
|
|
39
|
-
const stack = [];
|
|
40
|
-
this._currObj = {};
|
|
41
|
-
this._currKey = 'feed';
|
|
42
|
-
let inArray = false;
|
|
43
|
-
let feedJustFound = false;
|
|
44
|
-
|
|
45
|
-
// Look feed object in case this is a json encoded atom feed.
|
|
46
|
-
// Place these underlying keys onto the root object.
|
|
47
|
-
const findfeed = (key) => {
|
|
48
|
-
if (key === 'feed') {
|
|
49
|
-
feedJustFound = true;
|
|
50
|
-
parser.removeListener('openobject', findfeed);
|
|
51
|
-
parser.removeListener('key', findfeed);
|
|
52
|
-
}
|
|
53
|
-
};
|
|
54
|
-
|
|
55
|
-
const onvalue = (value) => {
|
|
56
|
-
feedJustFound = false;
|
|
57
|
-
this._currObj[this._currKey] = value;
|
|
58
|
-
if (stack.length === 1) {
|
|
59
|
-
parser.emit(this._currKey, value);
|
|
60
|
-
if (!buffer) { delete this._currObj[this._currKey]; }
|
|
61
|
-
}
|
|
62
|
-
if (inArray) {
|
|
63
|
-
this._currKey++;
|
|
64
|
-
}
|
|
65
|
-
};
|
|
66
|
-
|
|
67
|
-
const onopenobject = (key) => {
|
|
68
|
-
if (feedJustFound) {
|
|
69
|
-
feedJustFound = false;
|
|
70
|
-
this._currKey = key;
|
|
71
|
-
return;
|
|
72
|
-
}
|
|
73
|
-
let obj = this._currObj[this._currKey] = {};
|
|
74
|
-
stack.push({
|
|
75
|
-
obj: this._currObj,
|
|
76
|
-
key: this._currKey,
|
|
77
|
-
arr: inArray,
|
|
78
|
-
});
|
|
79
|
-
this._currObj = obj;
|
|
80
|
-
this._currKey = key;
|
|
81
|
-
inArray = false;
|
|
82
|
-
};
|
|
83
|
-
|
|
84
|
-
const onkey = (key) => { this._currKey = key; };
|
|
85
|
-
|
|
86
|
-
const oncloseobject = () => {
|
|
87
|
-
let parent = stack.pop();
|
|
88
|
-
if (!parent) { return; }
|
|
89
|
-
this._currObj = parent.obj;
|
|
90
|
-
this._currKey = parent.key;
|
|
91
|
-
inArray = parent.arr;
|
|
92
|
-
|
|
93
|
-
// Clean object.
|
|
94
|
-
this._currObj[this._currKey] = cleanObject(this._currObj[this._currKey]);
|
|
95
|
-
|
|
96
|
-
// Emit key in feed if curr is parent.
|
|
97
|
-
if (stack.length === 1) {
|
|
98
|
-
parser.emit(this._currKey, this._currObj[this._currKey]);
|
|
99
|
-
if (!buffer) { delete this._currObj[this._currKey]; }
|
|
100
|
-
|
|
101
|
-
// Or parent is array.
|
|
102
|
-
} else if (inArray) {
|
|
103
|
-
if (stack.length === 2) {
|
|
104
|
-
let key = stack[1].key;
|
|
105
|
-
let event = key === 'entry' || key === 'items' ?
|
|
106
|
-
'item' : stack[1].key;
|
|
107
|
-
let data = this._currObj[this._currKey];
|
|
108
|
-
parser.emit(event, data);
|
|
109
|
-
if (!buffer) { this._currObj.splice(this._currKey, 1); }
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
if (stack.length > 2 || buffer) { this._currKey++; }
|
|
113
|
-
}
|
|
114
|
-
};
|
|
115
|
-
|
|
116
|
-
const onopenarray = () => {
|
|
117
|
-
feedJustFound = false;
|
|
118
|
-
let obj = this._currObj[this._currKey] = [];
|
|
119
|
-
stack.push({
|
|
120
|
-
obj: this._currObj,
|
|
121
|
-
key: this._currKey,
|
|
122
|
-
arr: inArray,
|
|
123
|
-
});
|
|
124
|
-
this._currObj = obj;
|
|
125
|
-
this._currKey = 0;
|
|
126
|
-
inArray = true;
|
|
127
|
-
};
|
|
128
|
-
|
|
129
|
-
const onclosearray = () => {
|
|
130
|
-
let parent = stack.pop();
|
|
131
|
-
this._currObj = parent.obj;
|
|
132
|
-
this._currKey = parent.key;
|
|
133
|
-
inArray = parent.arr;
|
|
134
|
-
|
|
135
|
-
if (stack.length === 1) {
|
|
136
|
-
if (!buffer) { delete this._currObj[this._currKey]; }
|
|
137
|
-
} else if (inArray) {
|
|
138
|
-
this._currKey++;
|
|
139
|
-
}
|
|
140
|
-
};
|
|
141
|
-
|
|
142
|
-
parser.on('openobject', findfeed);
|
|
143
|
-
parser.on('key', findfeed);
|
|
144
|
-
parser.on('value', onvalue);
|
|
145
|
-
parser.on('openobject', onopenobject);
|
|
146
|
-
parser.on('key', onkey);
|
|
147
|
-
parser.on('closeobject', oncloseobject);
|
|
148
|
-
parser.on('openarray', onopenarray);
|
|
149
|
-
parser.on('closearray', onclosearray);
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
_write(chunk, encoding, callback) {
|
|
153
|
-
this.parser.write(chunk, encoding);
|
|
154
|
-
callback(null);
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
_final(callback) {
|
|
158
|
-
this.parser.end();
|
|
159
|
-
callback(null);
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
done() {
|
|
163
|
-
if (!this._buffer) { return; }
|
|
164
|
-
let root = this._currObj[this._currKey];
|
|
165
|
-
root.type = 'json';
|
|
166
|
-
if (Array.isArray(root.entry)) {
|
|
167
|
-
root.items = root.entry;
|
|
168
|
-
}
|
|
169
|
-
delete root.entry;
|
|
170
|
-
return root;
|
|
171
|
-
}
|
|
172
|
-
};
|
package/lib/xmlfeedparser.js
DELETED
|
@@ -1,197 +0,0 @@
|
|
|
1
|
-
const { Writable } = require('stream');
|
|
2
|
-
const sax = require('sax');
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
/**
|
|
6
|
-
* If this object has text, and there is no other children,
|
|
7
|
-
* or the only other key is type, make the object the text.
|
|
8
|
-
*
|
|
9
|
-
* @param {Object} obj
|
|
10
|
-
* @returns {Object}
|
|
11
|
-
*/
|
|
12
|
-
const cleanObj = (obj) => {
|
|
13
|
-
let keys = Object.keys(obj).length;
|
|
14
|
-
if (obj.text != null) {
|
|
15
|
-
// If this tag has text, trim it.
|
|
16
|
-
obj.text = trimIndent(obj.text);
|
|
17
|
-
|
|
18
|
-
if (keys === 1 || (keys === 2 && obj.type != null)) {
|
|
19
|
-
obj = obj.text;
|
|
20
|
-
}
|
|
21
|
-
} else if (keys === 0) {
|
|
22
|
-
obj = '';
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
return obj;
|
|
26
|
-
};
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
/**
|
|
30
|
-
* Trims the indent from text that sax-js emits. Some xml feeds will
|
|
31
|
-
* add indent to multiline text. Example:
|
|
32
|
-
*
|
|
33
|
-
* <item>
|
|
34
|
-
* The big brown fox
|
|
35
|
-
* jumped over the lazy dog
|
|
36
|
-
* </item>
|
|
37
|
-
*
|
|
38
|
-
* @param {string} str
|
|
39
|
-
* @returns {string}
|
|
40
|
-
*/
|
|
41
|
-
const trimIndent = (str) => {
|
|
42
|
-
let split = str.split('\n'), rs;
|
|
43
|
-
|
|
44
|
-
// The last line should be indented and only contain whitespace.
|
|
45
|
-
if (split.length > 1 &&
|
|
46
|
-
(rs = /^((?: {3})+|(?: {2})+|(?: {4})+|(?:\t)+)$/m.exec(
|
|
47
|
-
split[split.length - 1]))) {
|
|
48
|
-
|
|
49
|
-
// The very first line should be empty.
|
|
50
|
-
if (split.shift() !== '') {
|
|
51
|
-
return str;
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
// Remove the last line, as it's only whitespace.
|
|
55
|
-
split.pop();
|
|
56
|
-
|
|
57
|
-
let wholeindent = rs[0] + rs[1];
|
|
58
|
-
|
|
59
|
-
// Remove indent from beginning of every line.
|
|
60
|
-
for (let i = 0, l = split.length; i < l; i++) {
|
|
61
|
-
let s = split[i];
|
|
62
|
-
if (s.indexOf(wholeindent) !== 0) {
|
|
63
|
-
return str;
|
|
64
|
-
}
|
|
65
|
-
split[i] = s.substr(wholeindent.length);
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
str = split.join('\n');
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
return str;
|
|
72
|
-
};
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
/**
|
|
76
|
-
* Parses an RSS/Atom feed.
|
|
77
|
-
*
|
|
78
|
-
* @param {boolean} buffer If true, will buffer entire object.
|
|
79
|
-
* @return {sax.Stream}
|
|
80
|
-
*/
|
|
81
|
-
module.exports = class XMLFeedParser extends Writable {
|
|
82
|
-
constructor(buffer) {
|
|
83
|
-
super();
|
|
84
|
-
const parser = this.parser = sax.createStream(false, { lowercasetags: true });
|
|
85
|
-
const stack = [];
|
|
86
|
-
this._buffer = buffer;
|
|
87
|
-
this._obj = {};
|
|
88
|
-
|
|
89
|
-
// First start listening for the root tag.
|
|
90
|
-
const openf1 = (node) => {
|
|
91
|
-
if (node.name === 'channel' || node.name === 'feed') {
|
|
92
|
-
if (node.name === 'feed') {
|
|
93
|
-
let type = 'atom';
|
|
94
|
-
this._obj.type = type;
|
|
95
|
-
parser.emit('type', type);
|
|
96
|
-
}
|
|
97
|
-
parser.removeListener('opentag', openf1);
|
|
98
|
-
parser.on('text', ontext);
|
|
99
|
-
parser.on('cdata', ontext);
|
|
100
|
-
parser.on('opentag', onopentag);
|
|
101
|
-
parser.on('closetag', onclosetag);
|
|
102
|
-
|
|
103
|
-
} else if (node.name === 'rss' || node.name === 'rdf:rdf') {
|
|
104
|
-
let type = 'rss ' + (node.attributes.version || '1.0');
|
|
105
|
-
this._obj.type = type;
|
|
106
|
-
parser.emit('type', type);
|
|
107
|
-
} else {
|
|
108
|
-
parser.removeListener('opentag', openf1);
|
|
109
|
-
parser.emit('error', new Error('Feed type not recognized'));
|
|
110
|
-
parser.closed = true;
|
|
111
|
-
}
|
|
112
|
-
};
|
|
113
|
-
parser.on('opentag', openf1);
|
|
114
|
-
|
|
115
|
-
const ontext = (text) => {
|
|
116
|
-
// Make sure text events are text and not just whitespace.
|
|
117
|
-
const rs = /\n(( {3})+|( {2})+|( {4})+|(\t)+)$/m.exec(text);
|
|
118
|
-
if (rs !== null && rs.index === 0 && rs[0] === rs.input) { return; }
|
|
119
|
-
if (this._obj.text == null) { this._obj.text = ''; }
|
|
120
|
-
this._obj.text += text;
|
|
121
|
-
};
|
|
122
|
-
|
|
123
|
-
// After the root is found, start storing the rest of the document.
|
|
124
|
-
const onopentag = (node) => {
|
|
125
|
-
const key = node.name;
|
|
126
|
-
let i;
|
|
127
|
-
|
|
128
|
-
if (this._obj[key]) {
|
|
129
|
-
if (!Array.isArray(this._obj[key])) {
|
|
130
|
-
this._obj[key] = [this._obj[key], node.attributes];
|
|
131
|
-
i = 1;
|
|
132
|
-
} else {
|
|
133
|
-
i = this._obj[key].push(node.attributes) - 1;
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
} else {
|
|
137
|
-
this._obj[key] = node.attributes;
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
stack.push({ obj: this._obj, key, i });
|
|
141
|
-
this._obj = i ? this._obj[key][i] : this._obj[key];
|
|
142
|
-
};
|
|
143
|
-
|
|
144
|
-
const onclosetag = () => {
|
|
145
|
-
const parent = stack.pop();
|
|
146
|
-
if (!parent) { return; }
|
|
147
|
-
|
|
148
|
-
this._obj = parent.obj;
|
|
149
|
-
const key = parent.key;
|
|
150
|
-
const data = parent.i ?
|
|
151
|
-
this._obj[key][parent.i] = cleanObj(this._obj[key][parent.i]) :
|
|
152
|
-
this._obj[key] = cleanObj(this._obj[key]);
|
|
153
|
-
|
|
154
|
-
if (!stack.length) {
|
|
155
|
-
let skey = key === 'entry' || key === 'items' ? 'item' : key;
|
|
156
|
-
// Some feeds will contain a summary of items at the top.
|
|
157
|
-
// Ignore this.
|
|
158
|
-
if (key !== 'items' || Array.isArray(data)) {
|
|
159
|
-
parser.emit(skey, data);
|
|
160
|
-
if (!this._buffer) { delete this._obj[key]; }
|
|
161
|
-
}
|
|
162
|
-
}
|
|
163
|
-
};
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
_write(chunk, encoding, callback) {
|
|
167
|
-
this.parser.write(chunk, encoding);
|
|
168
|
-
callback(null);
|
|
169
|
-
}
|
|
170
|
-
|
|
171
|
-
_final(callback) {
|
|
172
|
-
this.parser.end();
|
|
173
|
-
callback(null);
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
// Called when done parsing the document.
|
|
177
|
-
// Returns entire document in object form.
|
|
178
|
-
done() {
|
|
179
|
-
if (!this._buffer) { return; }
|
|
180
|
-
let items;
|
|
181
|
-
if (this._obj.item != null) {
|
|
182
|
-
items = this._obj.item;
|
|
183
|
-
delete this._obj.item;
|
|
184
|
-
} else if (this._obj.entry != null) {
|
|
185
|
-
items = this._obj.entry;
|
|
186
|
-
delete this._obj.entry;
|
|
187
|
-
} else {
|
|
188
|
-
items = [];
|
|
189
|
-
}
|
|
190
|
-
if (!Array.isArray(items)){
|
|
191
|
-
items = [items];
|
|
192
|
-
}
|
|
193
|
-
this._obj.items = items;
|
|
194
|
-
delete this._obj.text;
|
|
195
|
-
return this._obj;
|
|
196
|
-
}
|
|
197
|
-
};
|