es6-crawler-detect 3.3.0 → 4.0.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.babelrc +1 -1
- package/.mocharc.json +5 -0
- package/README.md +1 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +18 -0
- package/dist/index.js.map +1 -0
- package/dist/lib/crawler/crawlers.d.ts +5 -0
- package/dist/lib/crawler/crawlers.d.ts.map +1 -0
- package/dist/lib/crawler/crawlers.js +1400 -0
- package/dist/lib/crawler/crawlers.js.map +1 -0
- package/dist/lib/crawler/exclusions.d.ts +5 -0
- package/dist/lib/crawler/exclusions.d.ts.map +1 -0
- package/dist/lib/crawler/exclusions.js +59 -0
- package/dist/lib/crawler/exclusions.js.map +1 -0
- package/dist/lib/crawler/headers.d.ts +5 -0
- package/dist/lib/crawler/headers.d.ts.map +1 -0
- package/dist/lib/crawler/headers.js +21 -0
- package/dist/lib/crawler/headers.js.map +1 -0
- package/dist/lib/crawler/provider.d.ts +4 -0
- package/dist/lib/crawler/provider.d.ts.map +1 -0
- package/dist/lib/crawler/provider.js +3 -0
- package/dist/lib/crawler/provider.js.map +1 -0
- package/dist/lib/crawler.d.ts +21 -0
- package/dist/lib/crawler.d.ts.map +1 -0
- package/dist/lib/crawler.js +98 -0
- package/dist/lib/crawler.js.map +1 -0
- package/dist/types.d.ts +9 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +3 -0
- package/dist/types.js.map +1 -0
- package/eslint.config.mjs +62 -0
- package/example/node/package.json +1 -1
- package/example/node/yarn.lock +222 -99
- package/package.json +30 -24
- package/src/index.ts +18 -0
- package/src/lib/crawler/{crawlers.js → crawlers.ts} +1397 -1403
- package/src/lib/crawler/{exclusions.js → exclusions.ts} +56 -62
- package/src/lib/crawler/{headers.js → headers.ts} +4 -10
- package/src/lib/crawler/provider.ts +3 -0
- package/src/lib/crawler.ts +128 -0
- package/src/types.ts +15 -0
- package/test/lib/crawler.spec.ts +116 -0
- package/tsconfig.json +28 -0
- package/.eslintignore +0 -6
- package/.eslintrc.json +0 -30
- package/dist/main.bundle.js +0 -1
- package/example/node/dist/crawler.min.js +0 -1
- package/example/node/node_modules/.bin/mime +0 -15
- package/example/node/node_modules/.bin/mime.cmd +0 -7
- package/example/node/node_modules/.yarn-integrity +0 -67
- package/example/node/node_modules/accepts/HISTORY.md +0 -236
- package/example/node/node_modules/accepts/LICENSE +0 -23
- package/example/node/node_modules/accepts/README.md +0 -142
- package/example/node/node_modules/accepts/index.js +0 -238
- package/example/node/node_modules/accepts/package.json +0 -47
- package/example/node/node_modules/array-flatten/LICENSE +0 -21
- package/example/node/node_modules/array-flatten/README.md +0 -43
- package/example/node/node_modules/array-flatten/array-flatten.js +0 -64
- package/example/node/node_modules/array-flatten/package.json +0 -39
- package/example/node/node_modules/body-parser/HISTORY.md +0 -624
- package/example/node/node_modules/body-parser/LICENSE +0 -23
- package/example/node/node_modules/body-parser/README.md +0 -456
- package/example/node/node_modules/body-parser/index.js +0 -157
- package/example/node/node_modules/body-parser/lib/read.js +0 -181
- package/example/node/node_modules/body-parser/lib/types/json.js +0 -230
- package/example/node/node_modules/body-parser/lib/types/raw.js +0 -101
- package/example/node/node_modules/body-parser/lib/types/text.js +0 -121
- package/example/node/node_modules/body-parser/lib/types/urlencoded.js +0 -284
- package/example/node/node_modules/body-parser/package.json +0 -52
- package/example/node/node_modules/bytes/History.md +0 -92
- package/example/node/node_modules/bytes/LICENSE +0 -23
- package/example/node/node_modules/bytes/Readme.md +0 -152
- package/example/node/node_modules/bytes/index.js +0 -166
- package/example/node/node_modules/bytes/package.json +0 -42
- package/example/node/node_modules/content-disposition/HISTORY.md +0 -60
- package/example/node/node_modules/content-disposition/LICENSE +0 -22
- package/example/node/node_modules/content-disposition/README.md +0 -142
- package/example/node/node_modules/content-disposition/index.js +0 -458
- package/example/node/node_modules/content-disposition/package.json +0 -44
- package/example/node/node_modules/content-type/HISTORY.md +0 -24
- package/example/node/node_modules/content-type/LICENSE +0 -22
- package/example/node/node_modules/content-type/README.md +0 -92
- package/example/node/node_modules/content-type/index.js +0 -222
- package/example/node/node_modules/content-type/package.json +0 -40
- package/example/node/node_modules/cookie/HISTORY.md +0 -128
- package/example/node/node_modules/cookie/LICENSE +0 -24
- package/example/node/node_modules/cookie/README.md +0 -257
- package/example/node/node_modules/cookie/index.js +0 -202
- package/example/node/node_modules/cookie/package.json +0 -40
- package/example/node/node_modules/cookie-signature/History.md +0 -38
- package/example/node/node_modules/cookie-signature/Readme.md +0 -42
- package/example/node/node_modules/cookie-signature/index.js +0 -51
- package/example/node/node_modules/cookie-signature/package.json +0 -18
- package/example/node/node_modules/debug/.coveralls.yml +0 -1
- package/example/node/node_modules/debug/.eslintrc +0 -11
- package/example/node/node_modules/debug/.travis.yml +0 -14
- package/example/node/node_modules/debug/CHANGELOG.md +0 -362
- package/example/node/node_modules/debug/LICENSE +0 -19
- package/example/node/node_modules/debug/Makefile +0 -50
- package/example/node/node_modules/debug/README.md +0 -312
- package/example/node/node_modules/debug/component.json +0 -19
- package/example/node/node_modules/debug/karma.conf.js +0 -70
- package/example/node/node_modules/debug/node.js +0 -1
- package/example/node/node_modules/debug/package.json +0 -49
- package/example/node/node_modules/debug/src/browser.js +0 -185
- package/example/node/node_modules/debug/src/debug.js +0 -202
- package/example/node/node_modules/debug/src/index.js +0 -10
- package/example/node/node_modules/debug/src/inspector-log.js +0 -15
- package/example/node/node_modules/debug/src/node.js +0 -248
- package/example/node/node_modules/depd/History.md +0 -96
- package/example/node/node_modules/depd/LICENSE +0 -22
- package/example/node/node_modules/depd/Readme.md +0 -280
- package/example/node/node_modules/depd/index.js +0 -522
- package/example/node/node_modules/depd/lib/browser/index.js +0 -77
- package/example/node/node_modules/depd/lib/compat/callsite-tostring.js +0 -103
- package/example/node/node_modules/depd/lib/compat/event-listener-count.js +0 -22
- package/example/node/node_modules/depd/lib/compat/index.js +0 -79
- package/example/node/node_modules/depd/package.json +0 -41
- package/example/node/node_modules/destroy/LICENSE +0 -22
- package/example/node/node_modules/destroy/README.md +0 -60
- package/example/node/node_modules/destroy/index.js +0 -75
- package/example/node/node_modules/destroy/package.json +0 -37
- package/example/node/node_modules/ee-first/LICENSE +0 -22
- package/example/node/node_modules/ee-first/README.md +0 -80
- package/example/node/node_modules/ee-first/index.js +0 -95
- package/example/node/node_modules/ee-first/package.json +0 -29
- package/example/node/node_modules/encodeurl/HISTORY.md +0 -14
- package/example/node/node_modules/encodeurl/LICENSE +0 -22
- package/example/node/node_modules/encodeurl/README.md +0 -128
- package/example/node/node_modules/encodeurl/index.js +0 -60
- package/example/node/node_modules/encodeurl/package.json +0 -40
- package/example/node/node_modules/escape-html/LICENSE +0 -24
- package/example/node/node_modules/escape-html/Readme.md +0 -43
- package/example/node/node_modules/escape-html/index.js +0 -78
- package/example/node/node_modules/escape-html/package.json +0 -24
- package/example/node/node_modules/etag/HISTORY.md +0 -83
- package/example/node/node_modules/etag/LICENSE +0 -22
- package/example/node/node_modules/etag/README.md +0 -159
- package/example/node/node_modules/etag/index.js +0 -131
- package/example/node/node_modules/etag/package.json +0 -47
- package/example/node/node_modules/express/History.md +0 -3510
- package/example/node/node_modules/express/LICENSE +0 -24
- package/example/node/node_modules/express/Readme.md +0 -158
- package/example/node/node_modules/express/index.js +0 -11
- package/example/node/node_modules/express/lib/application.js +0 -644
- package/example/node/node_modules/express/lib/express.js +0 -116
- package/example/node/node_modules/express/lib/middleware/init.js +0 -43
- package/example/node/node_modules/express/lib/middleware/query.js +0 -47
- package/example/node/node_modules/express/lib/request.js +0 -525
- package/example/node/node_modules/express/lib/response.js +0 -1147
- package/example/node/node_modules/express/lib/router/index.js +0 -668
- package/example/node/node_modules/express/lib/router/layer.js +0 -181
- package/example/node/node_modules/express/lib/router/route.js +0 -216
- package/example/node/node_modules/express/lib/utils.js +0 -302
- package/example/node/node_modules/express/lib/view.js +0 -182
- package/example/node/node_modules/express/package.json +0 -99
- package/example/node/node_modules/finalhandler/HISTORY.md +0 -187
- package/example/node/node_modules/finalhandler/LICENSE +0 -22
- package/example/node/node_modules/finalhandler/README.md +0 -148
- package/example/node/node_modules/finalhandler/index.js +0 -331
- package/example/node/node_modules/finalhandler/package.json +0 -45
- package/example/node/node_modules/forwarded/HISTORY.md +0 -21
- package/example/node/node_modules/forwarded/LICENSE +0 -22
- package/example/node/node_modules/forwarded/README.md +0 -57
- package/example/node/node_modules/forwarded/index.js +0 -90
- package/example/node/node_modules/forwarded/package.json +0 -45
- package/example/node/node_modules/fresh/HISTORY.md +0 -70
- package/example/node/node_modules/fresh/LICENSE +0 -23
- package/example/node/node_modules/fresh/README.md +0 -119
- package/example/node/node_modules/fresh/index.js +0 -137
- package/example/node/node_modules/fresh/package.json +0 -46
- package/example/node/node_modules/http-errors/HISTORY.md +0 -165
- package/example/node/node_modules/http-errors/LICENSE +0 -23
- package/example/node/node_modules/http-errors/README.md +0 -169
- package/example/node/node_modules/http-errors/index.js +0 -299
- package/example/node/node_modules/http-errors/package.json +0 -49
- package/example/node/node_modules/iconv-lite/Changelog.md +0 -162
- package/example/node/node_modules/iconv-lite/LICENSE +0 -21
- package/example/node/node_modules/iconv-lite/README.md +0 -156
- package/example/node/node_modules/iconv-lite/encodings/dbcs-codec.js +0 -555
- package/example/node/node_modules/iconv-lite/encodings/dbcs-data.js +0 -176
- package/example/node/node_modules/iconv-lite/encodings/index.js +0 -22
- package/example/node/node_modules/iconv-lite/encodings/internal.js +0 -188
- package/example/node/node_modules/iconv-lite/encodings/sbcs-codec.js +0 -72
- package/example/node/node_modules/iconv-lite/encodings/sbcs-data-generated.js +0 -451
- package/example/node/node_modules/iconv-lite/encodings/sbcs-data.js +0 -174
- package/example/node/node_modules/iconv-lite/encodings/tables/big5-added.json +0 -122
- package/example/node/node_modules/iconv-lite/encodings/tables/cp936.json +0 -264
- package/example/node/node_modules/iconv-lite/encodings/tables/cp949.json +0 -273
- package/example/node/node_modules/iconv-lite/encodings/tables/cp950.json +0 -177
- package/example/node/node_modules/iconv-lite/encodings/tables/eucjp.json +0 -182
- package/example/node/node_modules/iconv-lite/encodings/tables/gb18030-ranges.json +0 -1
- package/example/node/node_modules/iconv-lite/encodings/tables/gbk-added.json +0 -55
- package/example/node/node_modules/iconv-lite/encodings/tables/shiftjis.json +0 -125
- package/example/node/node_modules/iconv-lite/encodings/utf16.js +0 -177
- package/example/node/node_modules/iconv-lite/encodings/utf7.js +0 -290
- package/example/node/node_modules/iconv-lite/lib/bom-handling.js +0 -52
- package/example/node/node_modules/iconv-lite/lib/extend-node.js +0 -217
- package/example/node/node_modules/iconv-lite/lib/index.d.ts +0 -24
- package/example/node/node_modules/iconv-lite/lib/index.js +0 -153
- package/example/node/node_modules/iconv-lite/lib/streams.js +0 -121
- package/example/node/node_modules/iconv-lite/package.json +0 -46
- package/example/node/node_modules/inherits/LICENSE +0 -16
- package/example/node/node_modules/inherits/README.md +0 -42
- package/example/node/node_modules/inherits/inherits.js +0 -9
- package/example/node/node_modules/inherits/inherits_browser.js +0 -27
- package/example/node/node_modules/inherits/package.json +0 -29
- package/example/node/node_modules/ipaddr.js/LICENSE +0 -19
- package/example/node/node_modules/ipaddr.js/README.md +0 -233
- package/example/node/node_modules/ipaddr.js/ipaddr.min.js +0 -1
- package/example/node/node_modules/ipaddr.js/lib/ipaddr.js +0 -673
- package/example/node/node_modules/ipaddr.js/lib/ipaddr.js.d.ts +0 -68
- package/example/node/node_modules/ipaddr.js/package.json +0 -35
- package/example/node/node_modules/media-typer/HISTORY.md +0 -22
- package/example/node/node_modules/media-typer/LICENSE +0 -22
- package/example/node/node_modules/media-typer/README.md +0 -81
- package/example/node/node_modules/media-typer/index.js +0 -270
- package/example/node/node_modules/media-typer/package.json +0 -26
- package/example/node/node_modules/merge-descriptors/HISTORY.md +0 -21
- package/example/node/node_modules/merge-descriptors/LICENSE +0 -23
- package/example/node/node_modules/merge-descriptors/README.md +0 -48
- package/example/node/node_modules/merge-descriptors/index.js +0 -60
- package/example/node/node_modules/merge-descriptors/package.json +0 -32
- package/example/node/node_modules/methods/HISTORY.md +0 -29
- package/example/node/node_modules/methods/LICENSE +0 -24
- package/example/node/node_modules/methods/README.md +0 -51
- package/example/node/node_modules/methods/index.js +0 -69
- package/example/node/node_modules/methods/package.json +0 -36
- package/example/node/node_modules/mime/CHANGELOG.md +0 -164
- package/example/node/node_modules/mime/LICENSE +0 -21
- package/example/node/node_modules/mime/README.md +0 -90
- package/example/node/node_modules/mime/cli.js +0 -8
- package/example/node/node_modules/mime/mime.js +0 -108
- package/example/node/node_modules/mime/package.json +0 -44
- package/example/node/node_modules/mime/src/build.js +0 -53
- package/example/node/node_modules/mime/src/test.js +0 -60
- package/example/node/node_modules/mime/types.json +0 -1
- package/example/node/node_modules/mime-db/HISTORY.md +0 -499
- package/example/node/node_modules/mime-db/LICENSE +0 -22
- package/example/node/node_modules/mime-db/README.md +0 -100
- package/example/node/node_modules/mime-db/db.json +0 -8471
- package/example/node/node_modules/mime-db/index.js +0 -11
- package/example/node/node_modules/mime-db/package.json +0 -59
- package/example/node/node_modules/mime-types/HISTORY.md +0 -388
- package/example/node/node_modules/mime-types/LICENSE +0 -23
- package/example/node/node_modules/mime-types/README.md +0 -113
- package/example/node/node_modules/mime-types/index.js +0 -188
- package/example/node/node_modules/mime-types/package.json +0 -44
- package/example/node/node_modules/ms/index.js +0 -152
- package/example/node/node_modules/ms/license.md +0 -21
- package/example/node/node_modules/ms/package.json +0 -37
- package/example/node/node_modules/ms/readme.md +0 -51
- package/example/node/node_modules/negotiator/HISTORY.md +0 -103
- package/example/node/node_modules/negotiator/LICENSE +0 -24
- package/example/node/node_modules/negotiator/README.md +0 -203
- package/example/node/node_modules/negotiator/index.js +0 -124
- package/example/node/node_modules/negotiator/lib/charset.js +0 -169
- package/example/node/node_modules/negotiator/lib/encoding.js +0 -184
- package/example/node/node_modules/negotiator/lib/language.js +0 -179
- package/example/node/node_modules/negotiator/lib/mediaType.js +0 -294
- package/example/node/node_modules/negotiator/package.json +0 -42
- package/example/node/node_modules/on-finished/HISTORY.md +0 -88
- package/example/node/node_modules/on-finished/LICENSE +0 -23
- package/example/node/node_modules/on-finished/README.md +0 -154
- package/example/node/node_modules/on-finished/index.js +0 -196
- package/example/node/node_modules/on-finished/package.json +0 -31
- package/example/node/node_modules/parseurl/HISTORY.md +0 -58
- package/example/node/node_modules/parseurl/LICENSE +0 -24
- package/example/node/node_modules/parseurl/README.md +0 -133
- package/example/node/node_modules/parseurl/index.js +0 -158
- package/example/node/node_modules/parseurl/package.json +0 -40
- package/example/node/node_modules/path-to-regexp/History.md +0 -36
- package/example/node/node_modules/path-to-regexp/LICENSE +0 -21
- package/example/node/node_modules/path-to-regexp/Readme.md +0 -35
- package/example/node/node_modules/path-to-regexp/index.js +0 -129
- package/example/node/node_modules/path-to-regexp/package.json +0 -30
- package/example/node/node_modules/proxy-addr/HISTORY.md +0 -161
- package/example/node/node_modules/proxy-addr/LICENSE +0 -22
- package/example/node/node_modules/proxy-addr/README.md +0 -139
- package/example/node/node_modules/proxy-addr/index.js +0 -327
- package/example/node/node_modules/proxy-addr/package.json +0 -47
- package/example/node/node_modules/qs/.editorconfig +0 -39
- package/example/node/node_modules/qs/.eslintignore +0 -2
- package/example/node/node_modules/qs/.eslintrc +0 -35
- package/example/node/node_modules/qs/.github/FUNDING.yml +0 -12
- package/example/node/node_modules/qs/.nycrc +0 -13
- package/example/node/node_modules/qs/CHANGELOG.md +0 -359
- package/example/node/node_modules/qs/LICENSE.md +0 -29
- package/example/node/node_modules/qs/README.md +0 -604
- package/example/node/node_modules/qs/dist/qs.js +0 -832
- package/example/node/node_modules/qs/lib/formats.js +0 -23
- package/example/node/node_modules/qs/lib/index.js +0 -11
- package/example/node/node_modules/qs/lib/parse.js +0 -257
- package/example/node/node_modules/qs/lib/stringify.js +0 -278
- package/example/node/node_modules/qs/lib/utils.js +0 -251
- package/example/node/node_modules/qs/package.json +0 -69
- package/example/node/node_modules/qs/test/parse.js +0 -772
- package/example/node/node_modules/qs/test/stringify.js +0 -793
- package/example/node/node_modules/qs/test/utils.js +0 -136
- package/example/node/node_modules/range-parser/HISTORY.md +0 -56
- package/example/node/node_modules/range-parser/LICENSE +0 -23
- package/example/node/node_modules/range-parser/README.md +0 -84
- package/example/node/node_modules/range-parser/index.js +0 -162
- package/example/node/node_modules/range-parser/package.json +0 -44
- package/example/node/node_modules/raw-body/HISTORY.md +0 -284
- package/example/node/node_modules/raw-body/LICENSE +0 -22
- package/example/node/node_modules/raw-body/README.md +0 -217
- package/example/node/node_modules/raw-body/index.d.ts +0 -87
- package/example/node/node_modules/raw-body/index.js +0 -286
- package/example/node/node_modules/raw-body/package.json +0 -48
- package/example/node/node_modules/safe-buffer/LICENSE +0 -21
- package/example/node/node_modules/safe-buffer/README.md +0 -584
- package/example/node/node_modules/safe-buffer/index.d.ts +0 -187
- package/example/node/node_modules/safe-buffer/index.js +0 -65
- package/example/node/node_modules/safe-buffer/package.json +0 -51
- package/example/node/node_modules/safer-buffer/LICENSE +0 -21
- package/example/node/node_modules/safer-buffer/Porting-Buffer.md +0 -268
- package/example/node/node_modules/safer-buffer/Readme.md +0 -156
- package/example/node/node_modules/safer-buffer/dangerous.js +0 -58
- package/example/node/node_modules/safer-buffer/package.json +0 -34
- package/example/node/node_modules/safer-buffer/safer.js +0 -77
- package/example/node/node_modules/safer-buffer/tests.js +0 -406
- package/example/node/node_modules/send/HISTORY.md +0 -506
- package/example/node/node_modules/send/LICENSE +0 -23
- package/example/node/node_modules/send/README.md +0 -327
- package/example/node/node_modules/send/index.js +0 -1133
- package/example/node/node_modules/send/node_modules/.bin/mime +0 -15
- package/example/node/node_modules/send/node_modules/.bin/mime.cmd +0 -7
- package/example/node/node_modules/send/node_modules/ms/index.js +0 -162
- package/example/node/node_modules/send/node_modules/ms/license.md +0 -21
- package/example/node/node_modules/send/node_modules/ms/package.json +0 -38
- package/example/node/node_modules/send/node_modules/ms/readme.md +0 -59
- package/example/node/node_modules/send/package.json +0 -61
- package/example/node/node_modules/serve-static/HISTORY.md +0 -459
- package/example/node/node_modules/serve-static/LICENSE +0 -25
- package/example/node/node_modules/serve-static/README.md +0 -257
- package/example/node/node_modules/serve-static/index.js +0 -210
- package/example/node/node_modules/serve-static/package.json +0 -42
- package/example/node/node_modules/setprototypeof/LICENSE +0 -13
- package/example/node/node_modules/setprototypeof/README.md +0 -31
- package/example/node/node_modules/setprototypeof/index.d.ts +0 -2
- package/example/node/node_modules/setprototypeof/index.js +0 -17
- package/example/node/node_modules/setprototypeof/package.json +0 -38
- package/example/node/node_modules/setprototypeof/test/index.js +0 -24
- package/example/node/node_modules/statuses/HISTORY.md +0 -65
- package/example/node/node_modules/statuses/LICENSE +0 -23
- package/example/node/node_modules/statuses/README.md +0 -127
- package/example/node/node_modules/statuses/codes.json +0 -66
- package/example/node/node_modules/statuses/index.js +0 -113
- package/example/node/node_modules/statuses/package.json +0 -48
- package/example/node/node_modules/toidentifier/HISTORY.md +0 -9
- package/example/node/node_modules/toidentifier/LICENSE +0 -21
- package/example/node/node_modules/toidentifier/README.md +0 -61
- package/example/node/node_modules/toidentifier/index.js +0 -32
- package/example/node/node_modules/toidentifier/package.json +0 -38
- package/example/node/node_modules/type-is/HISTORY.md +0 -259
- package/example/node/node_modules/type-is/LICENSE +0 -23
- package/example/node/node_modules/type-is/README.md +0 -170
- package/example/node/node_modules/type-is/index.js +0 -266
- package/example/node/node_modules/type-is/package.json +0 -45
- package/example/node/node_modules/unpipe/HISTORY.md +0 -4
- package/example/node/node_modules/unpipe/LICENSE +0 -22
- package/example/node/node_modules/unpipe/README.md +0 -43
- package/example/node/node_modules/unpipe/index.js +0 -69
- package/example/node/node_modules/unpipe/package.json +0 -27
- package/example/node/node_modules/utils-merge/LICENSE +0 -20
- package/example/node/node_modules/utils-merge/README.md +0 -34
- package/example/node/node_modules/utils-merge/index.js +0 -23
- package/example/node/node_modules/utils-merge/package.json +0 -40
- package/example/node/node_modules/vary/HISTORY.md +0 -39
- package/example/node/node_modules/vary/LICENSE +0 -22
- package/example/node/node_modules/vary/README.md +0 -101
- package/example/node/node_modules/vary/index.js +0 -149
- package/example/node/node_modules/vary/package.json +0 -43
- package/example/node/server.js +0 -38
- package/src/index.js +0 -16
- package/src/lib/crawler/provider.js +0 -11
- package/src/lib/crawler.js +0 -130
- package/test/lib/crawler.test.js +0 -94
- package/webpack.common.js +0 -20
- package/webpack.dev.js +0 -7
- package/webpack.prod.js +0 -14
@@ -1,1133 +0,0 @@
|
|
1
|
-
/*!
|
2
|
-
* send
|
3
|
-
* Copyright(c) 2012 TJ Holowaychuk
|
4
|
-
* Copyright(c) 2014-2016 Douglas Christopher Wilson
|
5
|
-
* MIT Licensed
|
6
|
-
*/
|
7
|
-
|
8
|
-
'use strict'
|
9
|
-
|
10
|
-
/**
|
11
|
-
* Module dependencies.
|
12
|
-
* @private
|
13
|
-
*/
|
14
|
-
|
15
|
-
var createError = require('http-errors')
|
16
|
-
var debug = require('debug')('send')
|
17
|
-
var deprecate = require('depd')('send')
|
18
|
-
var destroy = require('destroy')
|
19
|
-
var encodeUrl = require('encodeurl')
|
20
|
-
var escapeHtml = require('escape-html')
|
21
|
-
var etag = require('etag')
|
22
|
-
var fresh = require('fresh')
|
23
|
-
var fs = require('fs')
|
24
|
-
var mime = require('mime')
|
25
|
-
var ms = require('ms')
|
26
|
-
var onFinished = require('on-finished')
|
27
|
-
var parseRange = require('range-parser')
|
28
|
-
var path = require('path')
|
29
|
-
var statuses = require('statuses')
|
30
|
-
var Stream = require('stream')
|
31
|
-
var util = require('util')
|
32
|
-
|
33
|
-
/**
|
34
|
-
* Path function references.
|
35
|
-
* @private
|
36
|
-
*/
|
37
|
-
|
38
|
-
var extname = path.extname
|
39
|
-
var join = path.join
|
40
|
-
var normalize = path.normalize
|
41
|
-
var resolve = path.resolve
|
42
|
-
var sep = path.sep
|
43
|
-
|
44
|
-
/**
|
45
|
-
* Regular expression for identifying a bytes Range header.
|
46
|
-
* @private
|
47
|
-
*/
|
48
|
-
|
49
|
-
var BYTES_RANGE_REGEXP = /^ *bytes=/
|
50
|
-
|
51
|
-
/**
|
52
|
-
* Maximum value allowed for the max age.
|
53
|
-
* @private
|
54
|
-
*/
|
55
|
-
|
56
|
-
var MAX_MAXAGE = 60 * 60 * 24 * 365 * 1000 // 1 year
|
57
|
-
|
58
|
-
/**
|
59
|
-
* Regular expression to match a path with a directory up component.
|
60
|
-
* @private
|
61
|
-
*/
|
62
|
-
|
63
|
-
var UP_PATH_REGEXP = /(?:^|[\\/])\.\.(?:[\\/]|$)/
|
64
|
-
|
65
|
-
/**
|
66
|
-
* Module exports.
|
67
|
-
* @public
|
68
|
-
*/
|
69
|
-
|
70
|
-
module.exports = send
|
71
|
-
module.exports.mime = mime
|
72
|
-
|
73
|
-
/**
|
74
|
-
* Return a `SendStream` for `req` and `path`.
|
75
|
-
*
|
76
|
-
* @param {object} req
|
77
|
-
* @param {string} path
|
78
|
-
* @param {object} [options]
|
79
|
-
* @return {SendStream}
|
80
|
-
* @public
|
81
|
-
*/
|
82
|
-
|
83
|
-
function send (req, path, options) {
|
84
|
-
return new SendStream(req, path, options)
|
85
|
-
}
|
86
|
-
|
87
|
-
/**
|
88
|
-
* Initialize a `SendStream` with the given `path`.
|
89
|
-
*
|
90
|
-
* @param {Request} req
|
91
|
-
* @param {String} path
|
92
|
-
* @param {object} [options]
|
93
|
-
* @private
|
94
|
-
*/
|
95
|
-
|
96
|
-
function SendStream (req, path, options) {
|
97
|
-
Stream.call(this)
|
98
|
-
|
99
|
-
var opts = options || {}
|
100
|
-
|
101
|
-
this.options = opts
|
102
|
-
this.path = path
|
103
|
-
this.req = req
|
104
|
-
|
105
|
-
this._acceptRanges = opts.acceptRanges !== undefined
|
106
|
-
? Boolean(opts.acceptRanges)
|
107
|
-
: true
|
108
|
-
|
109
|
-
this._cacheControl = opts.cacheControl !== undefined
|
110
|
-
? Boolean(opts.cacheControl)
|
111
|
-
: true
|
112
|
-
|
113
|
-
this._etag = opts.etag !== undefined
|
114
|
-
? Boolean(opts.etag)
|
115
|
-
: true
|
116
|
-
|
117
|
-
this._dotfiles = opts.dotfiles !== undefined
|
118
|
-
? opts.dotfiles
|
119
|
-
: 'ignore'
|
120
|
-
|
121
|
-
if (this._dotfiles !== 'ignore' && this._dotfiles !== 'allow' && this._dotfiles !== 'deny') {
|
122
|
-
throw new TypeError('dotfiles option must be "allow", "deny", or "ignore"')
|
123
|
-
}
|
124
|
-
|
125
|
-
this._hidden = Boolean(opts.hidden)
|
126
|
-
|
127
|
-
if (opts.hidden !== undefined) {
|
128
|
-
deprecate('hidden: use dotfiles: \'' + (this._hidden ? 'allow' : 'ignore') + '\' instead')
|
129
|
-
}
|
130
|
-
|
131
|
-
// legacy support
|
132
|
-
if (opts.dotfiles === undefined) {
|
133
|
-
this._dotfiles = undefined
|
134
|
-
}
|
135
|
-
|
136
|
-
this._extensions = opts.extensions !== undefined
|
137
|
-
? normalizeList(opts.extensions, 'extensions option')
|
138
|
-
: []
|
139
|
-
|
140
|
-
this._immutable = opts.immutable !== undefined
|
141
|
-
? Boolean(opts.immutable)
|
142
|
-
: false
|
143
|
-
|
144
|
-
this._index = opts.index !== undefined
|
145
|
-
? normalizeList(opts.index, 'index option')
|
146
|
-
: ['index.html']
|
147
|
-
|
148
|
-
this._lastModified = opts.lastModified !== undefined
|
149
|
-
? Boolean(opts.lastModified)
|
150
|
-
: true
|
151
|
-
|
152
|
-
this._maxage = opts.maxAge || opts.maxage
|
153
|
-
this._maxage = typeof this._maxage === 'string'
|
154
|
-
? ms(this._maxage)
|
155
|
-
: Number(this._maxage)
|
156
|
-
this._maxage = !isNaN(this._maxage)
|
157
|
-
? Math.min(Math.max(0, this._maxage), MAX_MAXAGE)
|
158
|
-
: 0
|
159
|
-
|
160
|
-
this._root = opts.root
|
161
|
-
? resolve(opts.root)
|
162
|
-
: null
|
163
|
-
|
164
|
-
if (!this._root && opts.from) {
|
165
|
-
this.from(opts.from)
|
166
|
-
}
|
167
|
-
}
|
168
|
-
|
169
|
-
/**
|
170
|
-
* Inherits from `Stream`.
|
171
|
-
*/
|
172
|
-
|
173
|
-
util.inherits(SendStream, Stream)
|
174
|
-
|
175
|
-
/**
|
176
|
-
* Enable or disable etag generation.
|
177
|
-
*
|
178
|
-
* @param {Boolean} val
|
179
|
-
* @return {SendStream}
|
180
|
-
* @api public
|
181
|
-
*/
|
182
|
-
|
183
|
-
SendStream.prototype.etag = deprecate.function(function etag (val) {
|
184
|
-
this._etag = Boolean(val)
|
185
|
-
debug('etag %s', this._etag)
|
186
|
-
return this
|
187
|
-
}, 'send.etag: pass etag as option')
|
188
|
-
|
189
|
-
/**
|
190
|
-
* Enable or disable "hidden" (dot) files.
|
191
|
-
*
|
192
|
-
* @param {Boolean} path
|
193
|
-
* @return {SendStream}
|
194
|
-
* @api public
|
195
|
-
*/
|
196
|
-
|
197
|
-
SendStream.prototype.hidden = deprecate.function(function hidden (val) {
|
198
|
-
this._hidden = Boolean(val)
|
199
|
-
this._dotfiles = undefined
|
200
|
-
debug('hidden %s', this._hidden)
|
201
|
-
return this
|
202
|
-
}, 'send.hidden: use dotfiles option')
|
203
|
-
|
204
|
-
/**
|
205
|
-
* Set index `paths`, set to a falsy
|
206
|
-
* value to disable index support.
|
207
|
-
*
|
208
|
-
* @param {String|Boolean|Array} paths
|
209
|
-
* @return {SendStream}
|
210
|
-
* @api public
|
211
|
-
*/
|
212
|
-
|
213
|
-
SendStream.prototype.index = deprecate.function(function index (paths) {
|
214
|
-
var index = !paths ? [] : normalizeList(paths, 'paths argument')
|
215
|
-
debug('index %o', paths)
|
216
|
-
this._index = index
|
217
|
-
return this
|
218
|
-
}, 'send.index: pass index as option')
|
219
|
-
|
220
|
-
/**
|
221
|
-
* Set root `path`.
|
222
|
-
*
|
223
|
-
* @param {String} path
|
224
|
-
* @return {SendStream}
|
225
|
-
* @api public
|
226
|
-
*/
|
227
|
-
|
228
|
-
SendStream.prototype.root = function root (path) {
|
229
|
-
this._root = resolve(String(path))
|
230
|
-
debug('root %s', this._root)
|
231
|
-
return this
|
232
|
-
}
|
233
|
-
|
234
|
-
SendStream.prototype.from = deprecate.function(SendStream.prototype.root,
|
235
|
-
'send.from: pass root as option')
|
236
|
-
|
237
|
-
SendStream.prototype.root = deprecate.function(SendStream.prototype.root,
|
238
|
-
'send.root: pass root as option')
|
239
|
-
|
240
|
-
/**
|
241
|
-
* Set max-age to `maxAge`.
|
242
|
-
*
|
243
|
-
* @param {Number} maxAge
|
244
|
-
* @return {SendStream}
|
245
|
-
* @api public
|
246
|
-
*/
|
247
|
-
|
248
|
-
SendStream.prototype.maxage = deprecate.function(function maxage (maxAge) {
|
249
|
-
this._maxage = typeof maxAge === 'string'
|
250
|
-
? ms(maxAge)
|
251
|
-
: Number(maxAge)
|
252
|
-
this._maxage = !isNaN(this._maxage)
|
253
|
-
? Math.min(Math.max(0, this._maxage), MAX_MAXAGE)
|
254
|
-
: 0
|
255
|
-
debug('max-age %d', this._maxage)
|
256
|
-
return this
|
257
|
-
}, 'send.maxage: pass maxAge as option')
|
258
|
-
|
259
|
-
/**
|
260
|
-
* Emit error with `status`.
|
261
|
-
*
|
262
|
-
* @param {number} status
|
263
|
-
* @param {Error} [err]
|
264
|
-
* @private
|
265
|
-
*/
|
266
|
-
|
267
|
-
SendStream.prototype.error = function error (status, err) {
|
268
|
-
// emit if listeners instead of responding
|
269
|
-
if (hasListeners(this, 'error')) {
|
270
|
-
return this.emit('error', createError(status, err, {
|
271
|
-
expose: false
|
272
|
-
}))
|
273
|
-
}
|
274
|
-
|
275
|
-
var res = this.res
|
276
|
-
var msg = statuses[status] || String(status)
|
277
|
-
var doc = createHtmlDocument('Error', escapeHtml(msg))
|
278
|
-
|
279
|
-
// clear existing headers
|
280
|
-
clearHeaders(res)
|
281
|
-
|
282
|
-
// add error headers
|
283
|
-
if (err && err.headers) {
|
284
|
-
setHeaders(res, err.headers)
|
285
|
-
}
|
286
|
-
|
287
|
-
// send basic response
|
288
|
-
res.statusCode = status
|
289
|
-
res.setHeader('Content-Type', 'text/html; charset=UTF-8')
|
290
|
-
res.setHeader('Content-Length', Buffer.byteLength(doc))
|
291
|
-
res.setHeader('Content-Security-Policy', "default-src 'none'")
|
292
|
-
res.setHeader('X-Content-Type-Options', 'nosniff')
|
293
|
-
res.end(doc)
|
294
|
-
}
|
295
|
-
|
296
|
-
/**
|
297
|
-
* Check if the pathname ends with "/".
|
298
|
-
*
|
299
|
-
* @return {boolean}
|
300
|
-
* @private
|
301
|
-
*/
|
302
|
-
|
303
|
-
SendStream.prototype.hasTrailingSlash = function hasTrailingSlash () {
|
304
|
-
return this.path[this.path.length - 1] === '/'
|
305
|
-
}
|
306
|
-
|
307
|
-
/**
|
308
|
-
* Check if this is a conditional GET request.
|
309
|
-
*
|
310
|
-
* @return {Boolean}
|
311
|
-
* @api private
|
312
|
-
*/
|
313
|
-
|
314
|
-
SendStream.prototype.isConditionalGET = function isConditionalGET () {
|
315
|
-
return this.req.headers['if-match'] ||
|
316
|
-
this.req.headers['if-unmodified-since'] ||
|
317
|
-
this.req.headers['if-none-match'] ||
|
318
|
-
this.req.headers['if-modified-since']
|
319
|
-
}
|
320
|
-
|
321
|
-
/**
|
322
|
-
* Check if the request preconditions failed.
|
323
|
-
*
|
324
|
-
* @return {boolean}
|
325
|
-
* @private
|
326
|
-
*/
|
327
|
-
|
328
|
-
SendStream.prototype.isPreconditionFailure = function isPreconditionFailure () {
|
329
|
-
var req = this.req
|
330
|
-
var res = this.res
|
331
|
-
|
332
|
-
// if-match
|
333
|
-
var match = req.headers['if-match']
|
334
|
-
if (match) {
|
335
|
-
var etag = res.getHeader('ETag')
|
336
|
-
return !etag || (match !== '*' && parseTokenList(match).every(function (match) {
|
337
|
-
return match !== etag && match !== 'W/' + etag && 'W/' + match !== etag
|
338
|
-
}))
|
339
|
-
}
|
340
|
-
|
341
|
-
// if-unmodified-since
|
342
|
-
var unmodifiedSince = parseHttpDate(req.headers['if-unmodified-since'])
|
343
|
-
if (!isNaN(unmodifiedSince)) {
|
344
|
-
var lastModified = parseHttpDate(res.getHeader('Last-Modified'))
|
345
|
-
return isNaN(lastModified) || lastModified > unmodifiedSince
|
346
|
-
}
|
347
|
-
|
348
|
-
return false
|
349
|
-
}
|
350
|
-
|
351
|
-
/**
|
352
|
-
* Strip content-* header fields.
|
353
|
-
*
|
354
|
-
* @private
|
355
|
-
*/
|
356
|
-
|
357
|
-
SendStream.prototype.removeContentHeaderFields = function removeContentHeaderFields () {
|
358
|
-
var res = this.res
|
359
|
-
var headers = getHeaderNames(res)
|
360
|
-
|
361
|
-
for (var i = 0; i < headers.length; i++) {
|
362
|
-
var header = headers[i]
|
363
|
-
if (header.substr(0, 8) === 'content-' && header !== 'content-location') {
|
364
|
-
res.removeHeader(header)
|
365
|
-
}
|
366
|
-
}
|
367
|
-
}
|
368
|
-
|
369
|
-
/**
|
370
|
-
* Respond with 304 not modified.
|
371
|
-
*
|
372
|
-
* @api private
|
373
|
-
*/
|
374
|
-
|
375
|
-
SendStream.prototype.notModified = function notModified () {
|
376
|
-
var res = this.res
|
377
|
-
debug('not modified')
|
378
|
-
this.removeContentHeaderFields()
|
379
|
-
res.statusCode = 304
|
380
|
-
res.end()
|
381
|
-
}
|
382
|
-
|
383
|
-
/**
|
384
|
-
* Raise error that headers already sent.
|
385
|
-
*
|
386
|
-
* @api private
|
387
|
-
*/
|
388
|
-
|
389
|
-
SendStream.prototype.headersAlreadySent = function headersAlreadySent () {
|
390
|
-
var err = new Error('Can\'t set headers after they are sent.')
|
391
|
-
debug('headers already sent')
|
392
|
-
this.error(500, err)
|
393
|
-
}
|
394
|
-
|
395
|
-
/**
|
396
|
-
* Check if the request is cacheable, aka
|
397
|
-
* responded with 2xx or 304 (see RFC 2616 section 14.2{5,6}).
|
398
|
-
*
|
399
|
-
* @return {Boolean}
|
400
|
-
* @api private
|
401
|
-
*/
|
402
|
-
|
403
|
-
SendStream.prototype.isCachable = function isCachable () {
|
404
|
-
var statusCode = this.res.statusCode
|
405
|
-
return (statusCode >= 200 && statusCode < 300) ||
|
406
|
-
statusCode === 304
|
407
|
-
}
|
408
|
-
|
409
|
-
/**
|
410
|
-
* Handle stat() error.
|
411
|
-
*
|
412
|
-
* @param {Error} error
|
413
|
-
* @private
|
414
|
-
*/
|
415
|
-
|
416
|
-
SendStream.prototype.onStatError = function onStatError (error) {
|
417
|
-
switch (error.code) {
|
418
|
-
case 'ENAMETOOLONG':
|
419
|
-
case 'ENOENT':
|
420
|
-
case 'ENOTDIR':
|
421
|
-
this.error(404, error)
|
422
|
-
break
|
423
|
-
default:
|
424
|
-
this.error(500, error)
|
425
|
-
break
|
426
|
-
}
|
427
|
-
}
|
428
|
-
|
429
|
-
/**
|
430
|
-
* Check if the cache is fresh.
|
431
|
-
*
|
432
|
-
* @return {Boolean}
|
433
|
-
* @api private
|
434
|
-
*/
|
435
|
-
|
436
|
-
SendStream.prototype.isFresh = function isFresh () {
|
437
|
-
return fresh(this.req.headers, {
|
438
|
-
etag: this.res.getHeader('ETag'),
|
439
|
-
'last-modified': this.res.getHeader('Last-Modified')
|
440
|
-
})
|
441
|
-
}
|
442
|
-
|
443
|
-
/**
|
444
|
-
* Check if the range is fresh.
|
445
|
-
*
|
446
|
-
* @return {Boolean}
|
447
|
-
* @api private
|
448
|
-
*/
|
449
|
-
|
450
|
-
SendStream.prototype.isRangeFresh = function isRangeFresh () {
|
451
|
-
var ifRange = this.req.headers['if-range']
|
452
|
-
|
453
|
-
if (!ifRange) {
|
454
|
-
return true
|
455
|
-
}
|
456
|
-
|
457
|
-
// if-range as etag
|
458
|
-
if (ifRange.indexOf('"') !== -1) {
|
459
|
-
var etag = this.res.getHeader('ETag')
|
460
|
-
return Boolean(etag && ifRange.indexOf(etag) !== -1)
|
461
|
-
}
|
462
|
-
|
463
|
-
// if-range as modified date
|
464
|
-
var lastModified = this.res.getHeader('Last-Modified')
|
465
|
-
return parseHttpDate(lastModified) <= parseHttpDate(ifRange)
|
466
|
-
}
|
467
|
-
|
468
|
-
/**
|
469
|
-
* Redirect to path.
|
470
|
-
*
|
471
|
-
* @param {string} path
|
472
|
-
* @private
|
473
|
-
*/
|
474
|
-
|
475
|
-
SendStream.prototype.redirect = function redirect (path) {
|
476
|
-
var res = this.res
|
477
|
-
|
478
|
-
if (hasListeners(this, 'directory')) {
|
479
|
-
this.emit('directory', res, path)
|
480
|
-
return
|
481
|
-
}
|
482
|
-
|
483
|
-
if (this.hasTrailingSlash()) {
|
484
|
-
this.error(403)
|
485
|
-
return
|
486
|
-
}
|
487
|
-
|
488
|
-
var loc = encodeUrl(collapseLeadingSlashes(this.path + '/'))
|
489
|
-
var doc = createHtmlDocument('Redirecting', 'Redirecting to <a href="' + escapeHtml(loc) + '">' +
|
490
|
-
escapeHtml(loc) + '</a>')
|
491
|
-
|
492
|
-
// redirect
|
493
|
-
res.statusCode = 301
|
494
|
-
res.setHeader('Content-Type', 'text/html; charset=UTF-8')
|
495
|
-
res.setHeader('Content-Length', Buffer.byteLength(doc))
|
496
|
-
res.setHeader('Content-Security-Policy', "default-src 'none'")
|
497
|
-
res.setHeader('X-Content-Type-Options', 'nosniff')
|
498
|
-
res.setHeader('Location', loc)
|
499
|
-
res.end(doc)
|
500
|
-
}
|
501
|
-
|
502
|
-
/**
|
503
|
-
* Pipe to `res.
|
504
|
-
*
|
505
|
-
* @param {Stream} res
|
506
|
-
* @return {Stream} res
|
507
|
-
* @api public
|
508
|
-
*/
|
509
|
-
|
510
|
-
SendStream.prototype.pipe = function pipe (res) {
|
511
|
-
// root path
|
512
|
-
var root = this._root
|
513
|
-
|
514
|
-
// references
|
515
|
-
this.res = res
|
516
|
-
|
517
|
-
// decode the path
|
518
|
-
var path = decode(this.path)
|
519
|
-
if (path === -1) {
|
520
|
-
this.error(400)
|
521
|
-
return res
|
522
|
-
}
|
523
|
-
|
524
|
-
// null byte(s)
|
525
|
-
if (~path.indexOf('\0')) {
|
526
|
-
this.error(400)
|
527
|
-
return res
|
528
|
-
}
|
529
|
-
|
530
|
-
var parts
|
531
|
-
if (root !== null) {
|
532
|
-
// normalize
|
533
|
-
if (path) {
|
534
|
-
path = normalize('.' + sep + path)
|
535
|
-
}
|
536
|
-
|
537
|
-
// malicious path
|
538
|
-
if (UP_PATH_REGEXP.test(path)) {
|
539
|
-
debug('malicious path "%s"', path)
|
540
|
-
this.error(403)
|
541
|
-
return res
|
542
|
-
}
|
543
|
-
|
544
|
-
// explode path parts
|
545
|
-
parts = path.split(sep)
|
546
|
-
|
547
|
-
// join / normalize from optional root dir
|
548
|
-
path = normalize(join(root, path))
|
549
|
-
} else {
|
550
|
-
// ".." is malicious without "root"
|
551
|
-
if (UP_PATH_REGEXP.test(path)) {
|
552
|
-
debug('malicious path "%s"', path)
|
553
|
-
this.error(403)
|
554
|
-
return res
|
555
|
-
}
|
556
|
-
|
557
|
-
// explode path parts
|
558
|
-
parts = normalize(path).split(sep)
|
559
|
-
|
560
|
-
// resolve the path
|
561
|
-
path = resolve(path)
|
562
|
-
}
|
563
|
-
|
564
|
-
// dotfile handling
|
565
|
-
if (containsDotFile(parts)) {
|
566
|
-
var access = this._dotfiles
|
567
|
-
|
568
|
-
// legacy support
|
569
|
-
if (access === undefined) {
|
570
|
-
access = parts[parts.length - 1][0] === '.'
|
571
|
-
? (this._hidden ? 'allow' : 'ignore')
|
572
|
-
: 'allow'
|
573
|
-
}
|
574
|
-
|
575
|
-
debug('%s dotfile "%s"', access, path)
|
576
|
-
switch (access) {
|
577
|
-
case 'allow':
|
578
|
-
break
|
579
|
-
case 'deny':
|
580
|
-
this.error(403)
|
581
|
-
return res
|
582
|
-
case 'ignore':
|
583
|
-
default:
|
584
|
-
this.error(404)
|
585
|
-
return res
|
586
|
-
}
|
587
|
-
}
|
588
|
-
|
589
|
-
// index file support
|
590
|
-
if (this._index.length && this.hasTrailingSlash()) {
|
591
|
-
this.sendIndex(path)
|
592
|
-
return res
|
593
|
-
}
|
594
|
-
|
595
|
-
this.sendFile(path)
|
596
|
-
return res
|
597
|
-
}
|
598
|
-
|
599
|
-
/**
|
600
|
-
* Transfer `path`.
|
601
|
-
*
|
602
|
-
* @param {String} path
|
603
|
-
* @api public
|
604
|
-
*/
|
605
|
-
|
606
|
-
SendStream.prototype.send = function send (path, stat) {
|
607
|
-
var len = stat.size
|
608
|
-
var options = this.options
|
609
|
-
var opts = {}
|
610
|
-
var res = this.res
|
611
|
-
var req = this.req
|
612
|
-
var ranges = req.headers.range
|
613
|
-
var offset = options.start || 0
|
614
|
-
|
615
|
-
if (headersSent(res)) {
|
616
|
-
// impossible to send now
|
617
|
-
this.headersAlreadySent()
|
618
|
-
return
|
619
|
-
}
|
620
|
-
|
621
|
-
debug('pipe "%s"', path)
|
622
|
-
|
623
|
-
// set header fields
|
624
|
-
this.setHeader(path, stat)
|
625
|
-
|
626
|
-
// set content-type
|
627
|
-
this.type(path)
|
628
|
-
|
629
|
-
// conditional GET support
|
630
|
-
if (this.isConditionalGET()) {
|
631
|
-
if (this.isPreconditionFailure()) {
|
632
|
-
this.error(412)
|
633
|
-
return
|
634
|
-
}
|
635
|
-
|
636
|
-
if (this.isCachable() && this.isFresh()) {
|
637
|
-
this.notModified()
|
638
|
-
return
|
639
|
-
}
|
640
|
-
}
|
641
|
-
|
642
|
-
// adjust len to start/end options
|
643
|
-
len = Math.max(0, len - offset)
|
644
|
-
if (options.end !== undefined) {
|
645
|
-
var bytes = options.end - offset + 1
|
646
|
-
if (len > bytes) len = bytes
|
647
|
-
}
|
648
|
-
|
649
|
-
// Range support
|
650
|
-
if (this._acceptRanges && BYTES_RANGE_REGEXP.test(ranges)) {
|
651
|
-
// parse
|
652
|
-
ranges = parseRange(len, ranges, {
|
653
|
-
combine: true
|
654
|
-
})
|
655
|
-
|
656
|
-
// If-Range support
|
657
|
-
if (!this.isRangeFresh()) {
|
658
|
-
debug('range stale')
|
659
|
-
ranges = -2
|
660
|
-
}
|
661
|
-
|
662
|
-
// unsatisfiable
|
663
|
-
if (ranges === -1) {
|
664
|
-
debug('range unsatisfiable')
|
665
|
-
|
666
|
-
// Content-Range
|
667
|
-
res.setHeader('Content-Range', contentRange('bytes', len))
|
668
|
-
|
669
|
-
// 416 Requested Range Not Satisfiable
|
670
|
-
return this.error(416, {
|
671
|
-
headers: { 'Content-Range': res.getHeader('Content-Range') }
|
672
|
-
})
|
673
|
-
}
|
674
|
-
|
675
|
-
// valid (syntactically invalid/multiple ranges are treated as a regular response)
|
676
|
-
if (ranges !== -2 && ranges.length === 1) {
|
677
|
-
debug('range %j', ranges)
|
678
|
-
|
679
|
-
// Content-Range
|
680
|
-
res.statusCode = 206
|
681
|
-
res.setHeader('Content-Range', contentRange('bytes', len, ranges[0]))
|
682
|
-
|
683
|
-
// adjust for requested range
|
684
|
-
offset += ranges[0].start
|
685
|
-
len = ranges[0].end - ranges[0].start + 1
|
686
|
-
}
|
687
|
-
}
|
688
|
-
|
689
|
-
// clone options
|
690
|
-
for (var prop in options) {
|
691
|
-
opts[prop] = options[prop]
|
692
|
-
}
|
693
|
-
|
694
|
-
// set read options
|
695
|
-
opts.start = offset
|
696
|
-
opts.end = Math.max(offset, offset + len - 1)
|
697
|
-
|
698
|
-
// content-length
|
699
|
-
res.setHeader('Content-Length', len)
|
700
|
-
|
701
|
-
// HEAD support
|
702
|
-
if (req.method === 'HEAD') {
|
703
|
-
res.end()
|
704
|
-
return
|
705
|
-
}
|
706
|
-
|
707
|
-
this.stream(path, opts)
|
708
|
-
}
|
709
|
-
|
710
|
-
/**
|
711
|
-
* Transfer file for `path`.
|
712
|
-
*
|
713
|
-
* @param {String} path
|
714
|
-
* @api private
|
715
|
-
*/
|
716
|
-
SendStream.prototype.sendFile = function sendFile (path) {
|
717
|
-
var i = 0
|
718
|
-
var self = this
|
719
|
-
|
720
|
-
debug('stat "%s"', path)
|
721
|
-
fs.stat(path, function onstat (err, stat) {
|
722
|
-
if (err && err.code === 'ENOENT' && !extname(path) && path[path.length - 1] !== sep) {
|
723
|
-
// not found, check extensions
|
724
|
-
return next(err)
|
725
|
-
}
|
726
|
-
if (err) return self.onStatError(err)
|
727
|
-
if (stat.isDirectory()) return self.redirect(path)
|
728
|
-
self.emit('file', path, stat)
|
729
|
-
self.send(path, stat)
|
730
|
-
})
|
731
|
-
|
732
|
-
function next (err) {
|
733
|
-
if (self._extensions.length <= i) {
|
734
|
-
return err
|
735
|
-
? self.onStatError(err)
|
736
|
-
: self.error(404)
|
737
|
-
}
|
738
|
-
|
739
|
-
var p = path + '.' + self._extensions[i++]
|
740
|
-
|
741
|
-
debug('stat "%s"', p)
|
742
|
-
fs.stat(p, function (err, stat) {
|
743
|
-
if (err) return next(err)
|
744
|
-
if (stat.isDirectory()) return next()
|
745
|
-
self.emit('file', p, stat)
|
746
|
-
self.send(p, stat)
|
747
|
-
})
|
748
|
-
}
|
749
|
-
}
|
750
|
-
|
751
|
-
/**
|
752
|
-
* Transfer index for `path`.
|
753
|
-
*
|
754
|
-
* @param {String} path
|
755
|
-
* @api private
|
756
|
-
*/
|
757
|
-
SendStream.prototype.sendIndex = function sendIndex (path) {
|
758
|
-
var i = -1
|
759
|
-
var self = this
|
760
|
-
|
761
|
-
function next (err) {
|
762
|
-
if (++i >= self._index.length) {
|
763
|
-
if (err) return self.onStatError(err)
|
764
|
-
return self.error(404)
|
765
|
-
}
|
766
|
-
|
767
|
-
var p = join(path, self._index[i])
|
768
|
-
|
769
|
-
debug('stat "%s"', p)
|
770
|
-
fs.stat(p, function (err, stat) {
|
771
|
-
if (err) return next(err)
|
772
|
-
if (stat.isDirectory()) return next()
|
773
|
-
self.emit('file', p, stat)
|
774
|
-
self.send(p, stat)
|
775
|
-
})
|
776
|
-
}
|
777
|
-
|
778
|
-
next()
|
779
|
-
}
|
780
|
-
|
781
|
-
/**
|
782
|
-
* Stream `path` to the response.
|
783
|
-
*
|
784
|
-
* @param {String} path
|
785
|
-
* @param {Object} options
|
786
|
-
* @api private
|
787
|
-
*/
|
788
|
-
|
789
|
-
SendStream.prototype.stream = function stream (path, options) {
|
790
|
-
// TODO: this is all lame, refactor meeee
|
791
|
-
var finished = false
|
792
|
-
var self = this
|
793
|
-
var res = this.res
|
794
|
-
|
795
|
-
// pipe
|
796
|
-
var stream = fs.createReadStream(path, options)
|
797
|
-
this.emit('stream', stream)
|
798
|
-
stream.pipe(res)
|
799
|
-
|
800
|
-
// response finished, done with the fd
|
801
|
-
onFinished(res, function onfinished () {
|
802
|
-
finished = true
|
803
|
-
destroy(stream)
|
804
|
-
})
|
805
|
-
|
806
|
-
// error handling code-smell
|
807
|
-
stream.on('error', function onerror (err) {
|
808
|
-
// request already finished
|
809
|
-
if (finished) return
|
810
|
-
|
811
|
-
// clean up stream
|
812
|
-
finished = true
|
813
|
-
destroy(stream)
|
814
|
-
|
815
|
-
// error
|
816
|
-
self.onStatError(err)
|
817
|
-
})
|
818
|
-
|
819
|
-
// end
|
820
|
-
stream.on('end', function onend () {
|
821
|
-
self.emit('end')
|
822
|
-
})
|
823
|
-
}
|
824
|
-
|
825
|
-
/**
|
826
|
-
* Set content-type based on `path`
|
827
|
-
* if it hasn't been explicitly set.
|
828
|
-
*
|
829
|
-
* @param {String} path
|
830
|
-
* @api private
|
831
|
-
*/
|
832
|
-
|
833
|
-
SendStream.prototype.type = function type (path) {
|
834
|
-
var res = this.res
|
835
|
-
|
836
|
-
if (res.getHeader('Content-Type')) return
|
837
|
-
|
838
|
-
var type = mime.lookup(path)
|
839
|
-
|
840
|
-
if (!type) {
|
841
|
-
debug('no content-type')
|
842
|
-
return
|
843
|
-
}
|
844
|
-
|
845
|
-
var charset = mime.charsets.lookup(type)
|
846
|
-
|
847
|
-
debug('content-type %s', type)
|
848
|
-
res.setHeader('Content-Type', type + (charset ? '; charset=' + charset : ''))
|
849
|
-
}
|
850
|
-
|
851
|
-
/**
|
852
|
-
* Set response header fields, most
|
853
|
-
* fields may be pre-defined.
|
854
|
-
*
|
855
|
-
* @param {String} path
|
856
|
-
* @param {Object} stat
|
857
|
-
* @api private
|
858
|
-
*/
|
859
|
-
|
860
|
-
SendStream.prototype.setHeader = function setHeader (path, stat) {
|
861
|
-
var res = this.res
|
862
|
-
|
863
|
-
this.emit('headers', res, path, stat)
|
864
|
-
|
865
|
-
if (this._acceptRanges && !res.getHeader('Accept-Ranges')) {
|
866
|
-
debug('accept ranges')
|
867
|
-
res.setHeader('Accept-Ranges', 'bytes')
|
868
|
-
}
|
869
|
-
|
870
|
-
if (this._cacheControl && !res.getHeader('Cache-Control')) {
|
871
|
-
var cacheControl = 'public, max-age=' + Math.floor(this._maxage / 1000)
|
872
|
-
|
873
|
-
if (this._immutable) {
|
874
|
-
cacheControl += ', immutable'
|
875
|
-
}
|
876
|
-
|
877
|
-
debug('cache-control %s', cacheControl)
|
878
|
-
res.setHeader('Cache-Control', cacheControl)
|
879
|
-
}
|
880
|
-
|
881
|
-
if (this._lastModified && !res.getHeader('Last-Modified')) {
|
882
|
-
var modified = stat.mtime.toUTCString()
|
883
|
-
debug('modified %s', modified)
|
884
|
-
res.setHeader('Last-Modified', modified)
|
885
|
-
}
|
886
|
-
|
887
|
-
if (this._etag && !res.getHeader('ETag')) {
|
888
|
-
var val = etag(stat)
|
889
|
-
debug('etag %s', val)
|
890
|
-
res.setHeader('ETag', val)
|
891
|
-
}
|
892
|
-
}
|
893
|
-
|
894
|
-
/**
|
895
|
-
* Clear all headers from a response.
|
896
|
-
*
|
897
|
-
* @param {object} res
|
898
|
-
* @private
|
899
|
-
*/
|
900
|
-
|
901
|
-
function clearHeaders (res) {
|
902
|
-
var headers = getHeaderNames(res)
|
903
|
-
|
904
|
-
for (var i = 0; i < headers.length; i++) {
|
905
|
-
res.removeHeader(headers[i])
|
906
|
-
}
|
907
|
-
}
|
908
|
-
|
909
|
-
/**
|
910
|
-
* Collapse all leading slashes into a single slash
|
911
|
-
*
|
912
|
-
* @param {string} str
|
913
|
-
* @private
|
914
|
-
*/
|
915
|
-
function collapseLeadingSlashes (str) {
|
916
|
-
for (var i = 0; i < str.length; i++) {
|
917
|
-
if (str[i] !== '/') {
|
918
|
-
break
|
919
|
-
}
|
920
|
-
}
|
921
|
-
|
922
|
-
return i > 1
|
923
|
-
? '/' + str.substr(i)
|
924
|
-
: str
|
925
|
-
}
|
926
|
-
|
927
|
-
/**
|
928
|
-
* Determine if path parts contain a dotfile.
|
929
|
-
*
|
930
|
-
* @api private
|
931
|
-
*/
|
932
|
-
|
933
|
-
function containsDotFile (parts) {
|
934
|
-
for (var i = 0; i < parts.length; i++) {
|
935
|
-
var part = parts[i]
|
936
|
-
if (part.length > 1 && part[0] === '.') {
|
937
|
-
return true
|
938
|
-
}
|
939
|
-
}
|
940
|
-
|
941
|
-
return false
|
942
|
-
}
|
943
|
-
|
944
|
-
/**
|
945
|
-
* Create a Content-Range header.
|
946
|
-
*
|
947
|
-
* @param {string} type
|
948
|
-
* @param {number} size
|
949
|
-
* @param {array} [range]
|
950
|
-
*/
|
951
|
-
|
952
|
-
function contentRange (type, size, range) {
|
953
|
-
return type + ' ' + (range ? range.start + '-' + range.end : '*') + '/' + size
|
954
|
-
}
|
955
|
-
|
956
|
-
/**
|
957
|
-
* Create a minimal HTML document.
|
958
|
-
*
|
959
|
-
* @param {string} title
|
960
|
-
* @param {string} body
|
961
|
-
* @private
|
962
|
-
*/
|
963
|
-
|
964
|
-
function createHtmlDocument (title, body) {
|
965
|
-
return '<!DOCTYPE html>\n' +
|
966
|
-
'<html lang="en">\n' +
|
967
|
-
'<head>\n' +
|
968
|
-
'<meta charset="utf-8">\n' +
|
969
|
-
'<title>' + title + '</title>\n' +
|
970
|
-
'</head>\n' +
|
971
|
-
'<body>\n' +
|
972
|
-
'<pre>' + body + '</pre>\n' +
|
973
|
-
'</body>\n' +
|
974
|
-
'</html>\n'
|
975
|
-
}
|
976
|
-
|
977
|
-
/**
|
978
|
-
* decodeURIComponent.
|
979
|
-
*
|
980
|
-
* Allows V8 to only deoptimize this fn instead of all
|
981
|
-
* of send().
|
982
|
-
*
|
983
|
-
* @param {String} path
|
984
|
-
* @api private
|
985
|
-
*/
|
986
|
-
|
987
|
-
function decode (path) {
|
988
|
-
try {
|
989
|
-
return decodeURIComponent(path)
|
990
|
-
} catch (err) {
|
991
|
-
return -1
|
992
|
-
}
|
993
|
-
}
|
994
|
-
|
995
|
-
/**
|
996
|
-
* Get the header names on a respnse.
|
997
|
-
*
|
998
|
-
* @param {object} res
|
999
|
-
* @returns {array[string]}
|
1000
|
-
* @private
|
1001
|
-
*/
|
1002
|
-
|
1003
|
-
function getHeaderNames (res) {
|
1004
|
-
return typeof res.getHeaderNames !== 'function'
|
1005
|
-
? Object.keys(res._headers || {})
|
1006
|
-
: res.getHeaderNames()
|
1007
|
-
}
|
1008
|
-
|
1009
|
-
/**
|
1010
|
-
* Determine if emitter has listeners of a given type.
|
1011
|
-
*
|
1012
|
-
* The way to do this check is done three different ways in Node.js >= 0.8
|
1013
|
-
* so this consolidates them into a minimal set using instance methods.
|
1014
|
-
*
|
1015
|
-
* @param {EventEmitter} emitter
|
1016
|
-
* @param {string} type
|
1017
|
-
* @returns {boolean}
|
1018
|
-
* @private
|
1019
|
-
*/
|
1020
|
-
|
1021
|
-
function hasListeners (emitter, type) {
|
1022
|
-
var count = typeof emitter.listenerCount !== 'function'
|
1023
|
-
? emitter.listeners(type).length
|
1024
|
-
: emitter.listenerCount(type)
|
1025
|
-
|
1026
|
-
return count > 0
|
1027
|
-
}
|
1028
|
-
|
1029
|
-
/**
|
1030
|
-
* Determine if the response headers have been sent.
|
1031
|
-
*
|
1032
|
-
* @param {object} res
|
1033
|
-
* @returns {boolean}
|
1034
|
-
* @private
|
1035
|
-
*/
|
1036
|
-
|
1037
|
-
function headersSent (res) {
|
1038
|
-
return typeof res.headersSent !== 'boolean'
|
1039
|
-
? Boolean(res._header)
|
1040
|
-
: res.headersSent
|
1041
|
-
}
|
1042
|
-
|
1043
|
-
/**
|
1044
|
-
* Normalize the index option into an array.
|
1045
|
-
*
|
1046
|
-
* @param {boolean|string|array} val
|
1047
|
-
* @param {string} name
|
1048
|
-
* @private
|
1049
|
-
*/
|
1050
|
-
|
1051
|
-
function normalizeList (val, name) {
|
1052
|
-
var list = [].concat(val || [])
|
1053
|
-
|
1054
|
-
for (var i = 0; i < list.length; i++) {
|
1055
|
-
if (typeof list[i] !== 'string') {
|
1056
|
-
throw new TypeError(name + ' must be array of strings or false')
|
1057
|
-
}
|
1058
|
-
}
|
1059
|
-
|
1060
|
-
return list
|
1061
|
-
}
|
1062
|
-
|
1063
|
-
/**
|
1064
|
-
* Parse an HTTP Date into a number.
|
1065
|
-
*
|
1066
|
-
* @param {string} date
|
1067
|
-
* @private
|
1068
|
-
*/
|
1069
|
-
|
1070
|
-
function parseHttpDate (date) {
|
1071
|
-
var timestamp = date && Date.parse(date)
|
1072
|
-
|
1073
|
-
return typeof timestamp === 'number'
|
1074
|
-
? timestamp
|
1075
|
-
: NaN
|
1076
|
-
}
|
1077
|
-
|
1078
|
-
/**
|
1079
|
-
* Parse a HTTP token list.
|
1080
|
-
*
|
1081
|
-
* @param {string} str
|
1082
|
-
* @private
|
1083
|
-
*/
|
1084
|
-
|
1085
|
-
function parseTokenList (str) {
|
1086
|
-
var end = 0
|
1087
|
-
var list = []
|
1088
|
-
var start = 0
|
1089
|
-
|
1090
|
-
// gather tokens
|
1091
|
-
for (var i = 0, len = str.length; i < len; i++) {
|
1092
|
-
switch (str.charCodeAt(i)) {
|
1093
|
-
case 0x20: /* */
|
1094
|
-
if (start === end) {
|
1095
|
-
start = end = i + 1
|
1096
|
-
}
|
1097
|
-
break
|
1098
|
-
case 0x2c: /* , */
|
1099
|
-
if (start !== end) {
|
1100
|
-
list.push(str.substring(start, end))
|
1101
|
-
}
|
1102
|
-
start = end = i + 1
|
1103
|
-
break
|
1104
|
-
default:
|
1105
|
-
end = i + 1
|
1106
|
-
break
|
1107
|
-
}
|
1108
|
-
}
|
1109
|
-
|
1110
|
-
// final token
|
1111
|
-
if (start !== end) {
|
1112
|
-
list.push(str.substring(start, end))
|
1113
|
-
}
|
1114
|
-
|
1115
|
-
return list
|
1116
|
-
}
|
1117
|
-
|
1118
|
-
/**
|
1119
|
-
* Set an object of headers on a response.
|
1120
|
-
*
|
1121
|
-
* @param {object} res
|
1122
|
-
* @param {object} headers
|
1123
|
-
* @private
|
1124
|
-
*/
|
1125
|
-
|
1126
|
-
function setHeaders (res, headers) {
|
1127
|
-
var keys = Object.keys(headers)
|
1128
|
-
|
1129
|
-
for (var i = 0; i < keys.length; i++) {
|
1130
|
-
var key = keys[i]
|
1131
|
-
res.setHeader(key, headers[key])
|
1132
|
-
}
|
1133
|
-
}
|