@xriss/weerss 0.0.4 → 0.0.6

package/cmd.js

@@ -2,6 +2,7 @@
 const cmd={}
 export default cmd
 
+import wtf from 'wtfnode'
 
 import mri from "mri"
 
@@ -98,6 +99,7 @@ if( arg1=="config" )
 else
 if( arg1=="fetch" )
 {
+	args.wtf=true // torrents are a problem so force exit afterwards
 	await weerss.fetch(args)
 }
 else
@@ -124,3 +126,10 @@ else
 {
 	console.log(` Unknown weerss command "${arg1}" `)
 }
+
+if(args.wtf)
+{
+	console.log(` Forcing exit `)	
+	wtf.dump()
+	process.exit(0)
+}

package/hoard.js

@@ -66,10 +66,13 @@ hoard.fetch_text=async function(url,refresh)
 		it.date=new Date()
 		try{
 
+			let timeout_id
 			let res=await Promise.race([
 				fetch(corsurl),
-				new Promise((_, reject) => setTimeout(() => reject("timeout"), 2*1000)),
+				new Promise(function(_, reject){ timeout_id=setTimeout(function(){reject("timeout")}, 2*1000) } ),
 			])
+			if(timeout_id) { clearTimeout(timeout_id) }
+
 			it.status=res.status
 			it.text=await res.text()
 			

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@xriss/weerss",
-	"version": "0.0.4",
+	"version": "0.0.6",
 	"source": "weerss.js",
 	"type": "module",
 	"repository": {
@@ -17,6 +17,7 @@
 	"dependencies": {
 		"@xriss/djon": "^0.0.14",
 		"express": "^4.19.2",
+		"memory-chunk-store": "^1.3.5",
 		"move-file": "^3.1.0",
 		"mri": "^1.2.0",
 		"parse-torrent": "^11.0.17",
@@ -27,7 +28,9 @@
 		"sax": "^1.4.1",
 		"sqlite3": "^5.1.7",
 		"stream-browserify": "^3.0.0",
-		"webpack": "^5.94.0"
+		"webpack": "^5.94.0",
+		"webtorrent": "^2.5.1",
+		"wtfnode": "^0.9.3"
 	},
 	"devDependencies": {
 		"webpack-cli": "^5.1.4"

package/torrents.js

@@ -4,9 +4,12 @@ let torrents={}
 export default torrents
 
 
-import      db        from "./db.js"
+import db from "./db.js"
+import hoard from "./hoard.js"
 
+import WebTorrent from 'webtorrent'
 import { remote } from 'parse-torrent'
+import memchunks from 'memory-chunk-store'
 
 
 
@@ -27,15 +30,90 @@ torrents.fetch=async function(url)
 	let ret = await torrents.get(url)
 	if( ret && ret.data) { return ret.data }
 	
-	let torrent_parse=function(torrent) {
+	let urlplus=url
+	if(urlplus.toLowerCase().startsWith("magnet:")) // auto add trackers
+	{
+		let trackers=(await hoard.fast_text("https://newtrackon.com/api/stable")).split("\n")
+		let count=0
+		for(let t of trackers)
+		{
+			t=t.trim()
+			if(t!="")
+			{
+				urlplus=urlplus+"&tr="+encodeURIComponent(t)
+				count=count+1
+				if(count>10){break}
+			}
+		}
+	}
+//	console.log(urlplus)
+	
+	let torrent_parse=function(url) {
 		return new Promise(function(resolve, reject) {
-			remote(torrent, { timeout: 60 * 1000 }, function(err, parsedTorrent){
+			remote(url, { timeout: 60 * 1000 }, function(err, parsedTorrent){
 				if (err) { reject(err) }
 				resolve(parsedTorrent)
 			})
 		})
 	}
-	let data=await torrent_parse(url) // should work with magnets
+	let data=await torrent_parse(urlplus) // does not work with magnets?
+
+	let client
+	let torrent_magnet_add=async function(url) {
+		return new Promise(function(resolve, reject)
+		{
+			client = new WebTorrent()
+			client.on('error', function(err){ if(!client.destroyed){client.destroy()} ; reject(err) })
+			// download to memory using memchunks
+			let torrent=client.add(url, {store:memchunks} , function(torrent){
+				resolve(torrent)
+			})
+			torrent.on('error', function(err){ if(!client.destroyed){client.destroy()} ; reject(err) })
+			torrent.on('noPeers', function(err){ if(!client.destroyed){client.destroy()} ; reject(err) })
+/*
+			for(let n of [
+				"infoHash" , "metadata" , "ready" , "warning" ,
+				"error" , "done" , "download" , "upload" ,
+				"wire" , "noPeers" , "verified"
+			])
+			{
+				torrent.on(n, function(d){ console.log(n,d) })
+			}
+*/
+		})
+	}
+	
+	if(!data.files) // need to magnet?
+	{
+		data.files=[]
+//		console.log(data)
+
+		let timeout_id
+		let torrent=await Promise.race([
+			torrent_magnet_add(urlplus),
+			new Promise(function(_, reject){ timeout_id=setTimeout(function(){reject("timeout")}, 10*1000) } ),
+		])
+		if(timeout_id) { clearTimeout(timeout_id) }
+		
+		if(torrent)
+		{
+			data.length=torrent.length
+	//		console.log(data.files)
+			for( let ff of torrent.files )
+			{
+				let f={}
+				f.name=ff.name
+				f.length=ff.length
+				data.files.push(f)
+			}
+			torrent.destroy()
+		}
+		if(client)
+		{
+			if(!client.destroyed){client.destroy()}
+			client=undefined
+		}
+	}
 
 	if(data)
 	{
@@ -91,7 +169,7 @@ torrents.fill_torrent=async function(item)
 		if( torrent ) // did we get something
 		{
 			item.torrent={fail:true}
-			
+						
 			for(let file of torrent.files)
 			{
 				let pct=Math.floor(100*file.length/torrent.length)

package/weerss.js

@@ -167,6 +167,10 @@ weerss.fetch=async function()
 	await db.setup()
 //	db.clear("hoard") // delete cache on startup
 
+// clear cached data
+	db.clear("torrents")
+	db.clear("hoard")
+
 	for(let feed of await db.list("feeds") )
 	{
 		let keep=false
@@ -350,10 +354,10 @@ weerss.save_rss=async function(args)
   "/rss/channel/language" : "en-us" ,
   "/rss/channel/lastBuildDate" : dates ,
   "/rss/channel/pubDate" : dates ,
-  "/rss/channel/description" : "RERSS" ,
-  "/rss/channel/generator" : "RERSS" ,
-  "/rss/channel/link" : "RERSS" ,
-  "/rss/channel/title" : "RERSS" ,
+  "/rss/channel/description" : "WEERSS" ,
+  "/rss/channel/generator" : "WEERSS" ,
+  "/rss/channel/link" : "WEERSS" ,
+  "/rss/channel/title" : "WEERSS" ,
   "/rss/channel/item" : rss_items ,
 }