@xriss/weerss 0.0.3 → 0.0.5

package/cmd.js

@@ -2,8 +2,9 @@
 const cmd={}
 export default cmd
 
+import wtf from 'wtfnode'
 
-import minimist from "minimist"
+import mri from "mri"
 
 import path from "path"
 import url from "url"
@@ -17,14 +18,16 @@ import items from "./items.js"
 
 import weerss from "./weerss.js"
 
-const args = minimist(process.argv.slice(2),{boolean:true})
+const args = mri(process.argv.slice(2))
+//console.log(args)
 
-args.config=args.config || process.env.WEERSS_CONFIG
+args.config=args.config || process.env.WEERSS_CONFIG || process.env.HOME+"/.config/weerss/config.djon"
 
 await weerss.load_config(args)
 
 let arg1=args._[0] || "help"
 let arg2=args._[1]
+let arg3=args._[3]
 
 if( arg1=="help" )
 {
@@ -42,30 +45,48 @@ options:
 
 		export WEERSS_CONFIG=/home/dave/weerss.config
 
+All options are loaded into this config structure with . used as an 
+object seperator and the value parsed as json.
+
+EG To choose the number of items in your rss feed from the command line 
+and thus overide the config file setting.
+
+	--rss.length=100
+or
+	--rss='{length:100}'
+
+With the later removing any other values in the rss object while the 
+former just changes the legth value.
+
 commands:
 
-tvrss config
-	Print current config.
+weerss config
+	Create a default to config file if we do not already have one and 
+	then print the current config. I recomend reading the default 
+	config file for comments about how to configure weerss.
 
-tvrss config FILENAME
-	Save current (default?) config to FILENAME.
+weerss config --force
+	Force saving of default config over current config.
 
-tvrss fetch
+weerss fetch
 	Fetch all feeds and scan files with tvmaze.
 
-tvrss list
+weerss list
 	List torrents in db.
 
-tvrss rss
+weerss rss
 	Save torrents as rss.
 
-tvrss move
-	Move downloaded chaotic files to an organised location.
+weerss move
+	Move downloaded chaotic files to an organised jellyfin style 
+	directory.
 
-tvrss dirs
-	Create a script to cleanup TV dirs using tvmaze.
+weerss dirs
+	Create a script to cleanup TV dirs using tvmaze. This does not do 
+	any cleanup just prints bash commands that you may wish to run 
+	later.
 
-tvrss help
+weerss help
 	Print this help message.
 
 `)
@@ -78,6 +99,7 @@ if( arg1=="config" )
 else
 if( arg1=="fetch" )
 {
+	args.wtf=true // torrents are a problem so force exit afterwards
 	await weerss.fetch(args)
 }
 else
@@ -104,3 +126,10 @@ else
 {
 	console.log(` Unknown weerss command "${arg1}" `)
 }
+
+if(args.wtf)
+{
+	console.log(` Forcing exit `)	
+	wtf.dump()
+	process.exit(0)
+}

package/db.js

@@ -29,7 +29,7 @@ db.name="weerss"
 db.setup=async function()
 {
 
-	db.handle = await AsyncDatabase.open( weerss.get_config_path("sqlite_filename") );
+	db.handle = await AsyncDatabase.open( weerss.get_config_path("sqlite") );
 
 //	db.handle.inner.on("trace", (sql) => console.log("[TRACE]", sql));
 

package/hoard.js

@@ -66,10 +66,13 @@ hoard.fetch_text=async function(url,refresh)
 		it.date=new Date()
 		try{
 
+			let timeout_id
 			let res=await Promise.race([
 				fetch(corsurl),
-				new Promise((_, reject) => setTimeout(() => reject("timeout"), 2*1000)),
+				new Promise(function(_, reject){ timeout_id=setTimeout(function(){reject("timeout")}, 2*1000) } ),
 			])
+			if(timeout_id) { clearTimeout(timeout_id) }
+
 			it.status=res.status
 			it.text=await res.text()
 			

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@xriss/weerss",
-	"version": "0.0.3",
+	"version": "0.0.5",
 	"source": "weerss.js",
 	"type": "module",
 	"repository": {
@@ -17,8 +17,9 @@
 	"dependencies": {
 		"@xriss/djon": "^0.0.14",
 		"express": "^4.19.2",
-		"minimist": "^1.2.8",
+		"memory-chunk-store": "^1.3.5",
 		"move-file": "^3.1.0",
+		"mri": "^1.2.0",
 		"parse-torrent": "^11.0.17",
 		"path-browserify": "^1.0.1",
 		"promised-sqlite3": "^2.1.0",
@@ -27,7 +28,9 @@
 		"sax": "^1.4.1",
 		"sqlite3": "^5.1.7",
 		"stream-browserify": "^3.0.0",
-		"webpack": "^5.94.0"
+		"webpack": "^5.94.0",
+		"webtorrent": "^2.5.1",
+		"wtfnode": "^0.9.3"
 	},
 	"devDependencies": {
 		"webpack-cli": "^5.1.4"

package/torrents.js

@@ -4,9 +4,12 @@ let torrents={}
 export default torrents
 
 
-import      db        from "./db.js"
+import db from "./db.js"
+import hoard from "./hoard.js"
 
+import WebTorrent from 'webtorrent'
 import { remote } from 'parse-torrent'
+import memchunks from 'memory-chunk-store'
 
 
 
@@ -27,15 +30,90 @@ torrents.fetch=async function(url)
 	let ret = await torrents.get(url)
 	if( ret && ret.data) { return ret.data }
 	
-	let torrent_parse=function(torrent) {
+	let urlplus=url
+	if(urlplus.toLowerCase().startsWith("magnet:")) // auto add trackers
+	{
+		let trackers=(await hoard.fast_text("https://newtrackon.com/api/stable")).split("\n")
+		let count=0
+		for(let t of trackers)
+		{
+			t=t.trim()
+			if(t!="")
+			{
+				urlplus=urlplus+"&tr="+encodeURIComponent(t)
+				count=count+1
+				if(count>10){break}
+			}
+		}
+	}
+//	console.log(urlplus)
+	
+	let torrent_parse=function(url) {
 		return new Promise(function(resolve, reject) {
-			remote(torrent, { timeout: 60 * 1000 }, function(err, parsedTorrent){
+			remote(url, { timeout: 60 * 1000 }, function(err, parsedTorrent){
 				if (err) { reject(err) }
 				resolve(parsedTorrent)
 			})
 		})
 	}
-	let data=await torrent_parse(url) // should work with magnets
+	let data=await torrent_parse(urlplus) // does not work with magnets?
+
+	let client
+	let torrent_magnet_add=async function(url) {
+		return new Promise(function(resolve, reject)
+		{
+			client = new WebTorrent()
+			client.on('error', function(err){ if(!client.destroyed){client.destroy()} ; reject(err) })
+			// download to memory using memchunks
+			let torrent=client.add(url, {store:memchunks} , function(torrent){
+				resolve(torrent)
+			})
+			torrent.on('error', function(err){ if(!client.destroyed){client.destroy()} ; reject(err) })
+			torrent.on('noPeers', function(err){ if(!client.destroyed){client.destroy()} ; reject(err) })
+/*
+			for(let n of [
+				"infoHash" , "metadata" , "ready" , "warning" ,
+				"error" , "done" , "download" , "upload" ,
+				"wire" , "noPeers" , "verified"
+			])
+			{
+				torrent.on(n, function(d){ console.log(n,d) })
+			}
+*/
+		})
+	}
+	
+	if(!data.files) // need to magnet?
+	{
+		data.files=[]
+//		console.log(data)
+
+		let timeout_id
+		let torrent=await Promise.race([
+			torrent_magnet_add(urlplus),
+			new Promise(function(_, reject){ timeout_id=setTimeout(function(){reject("timeout")}, 10*1000) } ),
+		])
+		if(timeout_id) { clearTimeout(timeout_id) }
+		
+		if(torrent)
+		{
+			data.length=torrent.length
+	//		console.log(data.files)
+			for( let ff of torrent.files )
+			{
+				let f={}
+				f.name=ff.name
+				f.length=ff.length
+				data.files.push(f)
+			}
+			torrent.destroy()
+		}
+		if(client)
+		{
+			if(!client.destroyed){client.destroy()}
+			client=undefined
+		}
+	}
 
 	if(data)
 	{
@@ -91,7 +169,7 @@ torrents.fill_torrent=async function(item)
 		if( torrent ) // did we get something
 		{
 			item.torrent={fail:true}
-			
+						
 			for(let file of torrent.files)
 			{
 				let pct=Math.floor(100*file.length/torrent.length)

package/weerss.js

@@ -17,31 +17,36 @@ import shows from "./shows.js"
 import util from "util"
 import {moveFile} from 'move-file'
 
+import { promises as pfs } from "fs"
 import fs from "fs"
 import path from "path"
 
 import sanitize from "sanitize-filename"
 
-
 import djon from "@xriss/djon"
 
 
 
 weerss.config_default=`
 {
+ // set to true for verbose rule checking
  debug = FALSE
+ // array of url to rss feeds of torrent files
  feeds = [
   "https://archive.org/services/collection-rss.php?collection=television_inbox"
  ]
  rss = {
+  // number of episodes to list in the rss files
   length = 100
  }
+ // decide which shows we like ( data from tvmaze ) 
  show = {
   rules = [
    [ TRUE ]
    [ "!english" FALSE ]
   ]
  }
+ // decide which episode we like tags come from filename 
  episode = {
   best = "small"
   maxsize = 4000000000
@@ -51,9 +56,13 @@ weerss.config_default=`
    [ "480p" FALSE ]
   ]
  }
- sqlite_filename = "./weerss.sqlite"
- download_dirname = "./download"
- tv_dirname = "./TV"
+ // paths can be relative to this config files location
+ paths = {
+  rss = "./weerss.rss"
+  sqlite = "./weerss.sqlite"
+  download = "./download"
+  tv = "./TV"
+ }
 }
 `
 
@@ -86,29 +95,37 @@ weerss.load_config=async function(args)
 	{
 		if(n!="_")
 		{
-			weerss.config[n]=args[n]
+			let va=args[n]
+			try{ va=djon.load(va) }catch(e){} // if can parse
+			let ns=n.split(".")
+			let f=ns.pop()
+			let b=weerss.config
+			for( let v of ns ) { b=b[v] } // sub sets
+			b[f]=va
 		}
 	}
 }
 
 weerss.save_config=async function(args)
 {
-	if( args._[1] )
+	if(args.config) // need a config location
 	{
-		let fname=args._[1]
-		console.log(` Saving config to `+fname)
-		if(args.config) // loading a config
+		if( (!args.force) && fs.existsSync(args.config) ) // must not already exist
 		{
-			djon.save_file(fname,weerss.config,"djon","strict")
+			console.log( "Config already exists : "+args.config )
+			console.log( djon.save(weerss.config,"djon","strict") )
 		}
-		else // save default
+		else // write defaukt config
 		{
-			fs.writeFileSync( fname , weerss.config_default )
+			console.log( "Writing config to : "+args.config )
+			try{ await pfs.mkdir( path.dirname(args.config) ) }catch(e){}
+			fs.writeFileSync( args.config , weerss.config_default )
 		}
-		return
 	}
-
-	console.log( djon.save(weerss.config,"djon","strict") )
+	else
+	{
+		console.log( "Must specify config file with --config=~/.weerss.config " )
+	}
 }
 
 weerss.get_config_path=function(name)
@@ -123,8 +140,8 @@ weerss.get_config_path=function(name)
 		d=path.resolve( "." )
 	}
 
-	let f=weerss.config[name] || "."
-	if( ! f.startsWith("/" ) )
+	let f=weerss.config.paths[name] || "."
+	if( ! path.isAbsolute(f) )
 	{
 		f=path.join( d , f )
 	}
@@ -313,7 +330,7 @@ weerss.list=async function(args)
 {
 	await db.setup()
 
-	let list=(await weerss.getlist())
+	let list=(await weerss.getlist()).slice(0,weerss.config.rss.length)
 	for(let item of list)
 	{
 		console.log( item_to_string(item))
@@ -367,16 +384,16 @@ weerss.save_rss=async function(args)
 		rss_items.push(it)
 	}
 
-	let list=(await weerss.getlist(weerss.config.show.rules)).slice(0,100)
+	let list=(await weerss.getlist(weerss.config.show.rules)).slice(0,weerss.config.rss.length)
 	for(let item of list)
 	{
 		push_item(item)
 	}
 
 
-	if( args._[1] )
+	let fname=weerss.get_config_path("rss")
+	if( fname )
 	{
-		let fname=args._[1]
 		console.log(` Saving rss to `+fname)
 
 		fs.writeFileSync( fname , jxml.build_xml(dat) )
@@ -393,8 +410,8 @@ weerss.save_rss=async function(args)
 weerss.move=async function(args)
 {
 	
-	let from=weerss.get_config_path("download_dirname") //args._[2] // optional dest dir
-	let dest=weerss.get_config_path("tv_dirname") //args._[2] // optional dest dir
+	let from=weerss.get_config_path("download") //args._[2] // optional dest dir
+	let dest=weerss.get_config_path("tv") //args._[2] // optional dest dir
 	
 	dest=null // do not move
 	
@@ -483,7 +500,7 @@ weerss.move=async function(args)
 
 weerss.dirs=async function(args)
 {	
-	let dest=weerss.get_config_path("tv_dirname") 
+	let dest=weerss.get_config_path("tv") 
 
 	
 	await db.setup()