@jayesol/jayeson.lib.delivery 2.0.7-beta.0 → 2.0.7

package/lib/Subscriber.js

@@ -1,1083 +1,1083 @@
-var coreMsges = require("./CoreMessages")
-var deliveryStack = require("./DeliveryStack")
-var _ = require("underscore")
-var EventEmitter = require("eventemitter3");
-
-var streamFinder = require("@jayesol/jayeson.lib.streamfinder");
-var Discoverer = streamFinder.Discoverer;
-
-var util = require("./util")
-var _c = util._c
-
-var DefaultClient = deliveryStack.DefaultClient
-var EPEvent = deliveryStack.EPEvent
-var EPConnectedEvent = deliveryStack.EPConnectedEvent
-var EPConnectionError = deliveryStack.EPConnectionError
-var EPDisconnectedEvent = deliveryStack.EPDisconnectedEvent
-var IEndPointEventSource = deliveryStack.IEndPointEventSource
-var generateUUID = deliveryStack.generateUUID;
-var Util = deliveryStack.Util;
-var MessageWrapper = deliveryStack.MessageWrapper;
-
-var StreamRegistryMessageGroup = coreMsges.StreamRegistryMessageGroup;
-var	EmptyMessageClass = coreMsges.EmptyMessageClass;
-var	StringMessageClass = coreMsges.StringMessageClass;
-var	JSonMessageClass = coreMsges.JSonMessageClass;
-var	StreamRegistryRequest = coreMsges.StreamRegistryRequest;
-var	StreamRegistryResponse = coreMsges.StreamRegistryResponse;
-var	AuthGroup = coreMsges.AuthGroup;
-var	GenericMessageGroup = coreMsges.GenericMessageGroup;
-var	KeepAliveMessageGroup = coreMsges.KeepAliveMessageGroup;
-var TicketRenew = coreMsges.TicketRenew;
-
-/**
- * @author Ryan
- * 
- * This depends on external libraries: underscorejs, eventemitter, promisejs
- * 
- * This depends on internal libraries: coremessage, deliverystack, jsdiscoverer
- * 
- * The subscriber has the following features:
- * TODO: FILL IN THE FEATURES 
- * 
- */
-
-/**
- * Represents a configuration being passed
- * into the subscriber. In the javascript context,
- * 
- * The discovery pattern has the following format
- * {
- * 		id1 : [pattern 1, pattern 2 .... ],
- * 		id2 : [pattern A, pattern B .... ]			
- * }
- * 
- * example
- * 
- * {
- * 		8 : ["*_LIVE", "SBO_*"],	// price feed
- * 		7 : ["order_ryan"]			// order data feed for user ryan
- * }
- * 
- * The processor setup is as following:
- * {
- * 		id1 : processor 1 of type IMessageGroupProcessor,
- * 		id2 : processor 2 of type IMessageGroupProcessor
- * 
- * }
- * 
- */
-function SubscriberConfig() {
-	this.autoDiscover = true;	// by default, enable auto discovery
-	this.streamFinder = "";		// the URL to connect to the stream finder
-	this.username = "";			// if we want to login directly to the stream finder
-	this.password = "";
-	this.discoveryPattern = {};
-	this.processors = {};
-	this.schemes = ["ws"];		// by default we consume from ws
-	this.ticketRenewalInterval = 14*60*1000;	// default to 14 minutes
-	this.reconnectionInterval = 3000;			// reconnection to a relay node
-	this.exclusionThreshold = 3;				// the number of reconnection failed before the relay node is added to the excluded list
-	this.feedScope = "";
-	this.discoverByScope = true; //new version will discover by scope
-}
-
-/**
- * Represents a data node. This encapsulate the following information
- * <li>The client used to connect to the node</li>
- * <li>The stream being consumed from the node</li>
- * 
- * It also provides functionalities such as reconnection. It listen to ticket being renewed
- * by the subscriber and send the ticket to the server accordingly.
- * 
- * @params
- * 	<li>id - id of the data node</li>
- * 	<li>streamInfo - the first stream information used for connection<br/>
- *	<pre> 
- * 	{
- *		"group": "<groupId>",
- *		"stream": "<streamName>",
- *		"level": <level>,
- *		"serviceId" : <serviceId>,
- *		"connection": {
- *   		"protocol": "<protocol>",
- *   		"hostname": "<hostname>",
- *   		"port": <port>
- * 		}
- *	}
- *	</pre>
- *	</li>
- *	<li>clientId - id of the client</li>
- *	<li>ticket - ticket of the subscriber</li>
- *	<li>eventBus - an Event Emitter which is responsible to propagate events from DataNode</li>
- *	<li>subscriberConfig - configuration of the subscriber that created this data node</li>
- */
-var DATA_NODE_COUNTER = 0;
-
-function DataNode(id, clientId, ticket, streamInfo, eventBus, subscriberConfig) {
-	this.client = null;
-	/**
-	 * This data structure has the following form:
-	 * { group -> [stream information] }
-	 */
-	this.streams = {};
-	this.id = id;		// id of the data node
-	
-	this.counter = DATA_NODE_COUNTER++;
-	console.log("[DataNode] Node ", id, " counter ", this.counter, " is created!");
-	
-	/**
-	 * Current state of the data node
-	 */
-	this.state = DataNode.State.CREATED;
-	this.connection = streamInfo.connection;
-	
-	this.toBeConsumed = [];
-	
-	this.consumeStream(streamInfo);
-	
-	this.clientId = clientId;
-	this.ticket = ticket;
-	
-	this.consumeTaskTimeoutID = 0;
-	
-	this.eventBus = eventBus;
-	
-	this.sConf = subscriberConfig;
-	
-	// the list of promises which will be resolved when stop consumption messages
-	// for a stream is handled and replied by the server. The promise
-	// shall be successful if there is disconnection while waiting for the reply
-	// Format: streamId -> promiseObject e.g. "8_CROWN"
-	// the promise value is true / false
-	this.removalPromises = {};
-	// the list of promises which will be resolved when consumption messages
-	// for a stream is handled and replied by the server. The promise shall
-	// resolve to failed if there is disconnection waiting for the reply. The promise
-	// is created when the registration (not consumption) for the stream starts.
-	// The promise value is true / false.
-	this.consumptionPromises = {};
-	
-};
-DataNode.State = {
-	CREATED : 1,
-	CONNECTING : 2,
-	CONNECTED : 3,
-	RECONNECTING : 4,
-	DISCONNECTED : 5,
-	DESTROYED : 6
-};
-DataNode.Event = {
-	STATE : "DN_STATE",
-	RECONNECTION_ERROR : "DN_RE_ERROR",
-	CONSUMPTION_ERROR : "DN_CONSUMPTION_ERROR",
-	CONSUMPTION_START : "DN_CONSUMPTION_START",
-	CONSUMPTION_STOP : "DN_CONSUMPTION_STOP",
-};
-_c(DataNode, {
-	
-	/**
-	 * This method return a PROMISE of the removal of the stream from the 
-	 * from consumption. This has to be asynchronous because involving sending
-	 * deregistration message to the stream registry of the consuming node. We however, only
-	 * send unregister when the node is in CONNECTED STATE and the person invoke this method
-	 * specify he want to do so (sendDeregistration = true)
-	 * 
-	 * There are two cases that a stream can be removed from a data node:
-	 * <li>The node that consuming the stream has an error and needs reconnection</li>
-	 * <li>A better node can be found for the stream according to the stream selection strategy</li>
-	 * 
-	 * @return a promise which is resolved when the stream is completely removed from consumption.
-	 */
-	removeStream : function(group, stream, sendDeregistration) {
-		
-		var id = Util._streamIdFromNames(group, stream);
-		
-		var deregistrationAction = _.bind(function(registrationResult) {
-			// the unregistration promise. This is done when the
-			// the stream is unregister from this data node by sending message to
-			// the data source
-			var promise;
-			
-			// the action to be done, remove the stream from the array
-			var unregisterPostAction = _.bind(function(result) {
-				if (result && this.streams[group]) {
-					var index;
-					
-					for (index = 0; index < this.streams[group].length; index++)
-						if (stream == this.streams[group][index].stream) break;
-					
-					if (index < this.streams[group].length) this.streams[group].splice(index, 1);
-					
-					if (this.streams[group].length == 0) delete this.streams[group];
-					
-					// if there is no more stream this node is consuming, just destroy itself
-					if (_.isEmpty(this.streams)) this.destroy();
-				}
-				
-				return result;
-			}, this);
-			
-			if (this.state == DataNode.State.CONNECTED && sendDeregistration) {
-				
-				
-				promise = Util._newPromise();
-				
-				
-				this.removalPromises[id] = promise;
-				
-				var req = new StreamRegistryRequest(group, [stream], StreamRegistryRequest.DEREGISTRATION);
-				this.client.send(new MessageWrapper(req, StreamRegistryMessageGroup.REQUEST));
-
-				var self = this;
-				window.setTimeout(function() {
-					//proceed with deregistration if no deregistration responses are received.
-					if (self.removalPromises[id] !== undefined) {
-						console.warn("Timeout waiting for deregistration response, proceeding to treat stream " + group + ":" + stream + " as deregistered for " + self.connection.hostname + ":" + self.connection.port);
-						var streamResponse = {};
-						streamResponse[stream] = StreamRegistryResponse.DEREGISTRATION_TIMEOUT;
-						var response = new StreamRegistryResponse(group, streamResponse, StreamRegistryResponse.DEREGISTRATION);
-
-						self._processDeregistrationResult(response);
-					}
-				}, 7000);
-				
-				// chain the promise
-				return promise.then(unregisterPostAction);
-				
-			} else {
-				// if the DataNode is not in connected state, no need to send anything!
-				promise = Promise.resolve(unregisterPostAction(true));
-				return promise;
-			}
-		}, this);
-		
-		// if there is a consumption in progress, wait for it! We will de-register the stream after the it finish consumption
-		// de-registration only happens in failing over or switching to better stream, so it's okay to de-register
-		// even when the pending consumption is successful. We switch anyway.
-		if (this.consumptionPromises && this.consumptionPromises[id] !== undefined) {	
-			return this.consumptionPromises[id].then(deregistrationAction);
-		} else {
-			return deregistrationAction(true);
-		}
-
-	},
-	
-	hasStream : function(group, stream) {
-		if (!this.streams[group]) {
-			return false;
-		}
-		var stream = _.each(this.streams[group], function(node) { return node.stream == stream; } );
-		return stream !== undefined;
-	},
-	
-	/**
-	 * Start consumption of a particular stream by putting it into tobeConsumed queue.
-	 * When a stream is consumed successfully, DataNode will emit an event
-	 * so that other parties know about it.
-	 */
-	consumeStream : function(streamInfo) {
-		
-		console.log("Node "+this.id+" is going to consume "+streamInfo.group+"_"+streamInfo.stream);
-		
-		this.toBeConsumed.push(streamInfo);
-		
-		if (!this.streams[streamInfo.group]) this.streams[streamInfo.group] = [];
-		
-		this.streams[streamInfo.group].push(streamInfo);	// put the stream into the list of streams that this DataNode handling
-		
-		if (this.consumeTaskTimeoutID == 0 && this.state == DataNode.State.CONNECTED)
-			this.consumeTaskTimeoutID = window.setTimeout(_.bind(this.consumeTask, this), 100);
-	},
-	
-	_consumeStreamsOnReconnected : function(){
-		
-		if (_.isEmpty(this.streams)) this.destroy();
-		
-		for(var group in this.streams){
-			if(this.streams.hasOwnProperty(group)){
-				for(var i=0 ; i<this.streams[group].length; i++){
-					this.toBeConsumed.push(this.streams[group][i]);
-				}
-			}
-		}
-	},
-	
-	connect : function() {
-		
-		console.log("[DataNode] Id ", this.id, " counter ", this.counter, " is going to connect. Current state ", this.state);
-		
-		//temp code
-		//this.connection.hostname = "local.vodds.jayeson.com.sg";
-		//this.connection.port = 5000;
-
-		this.client = new DefaultClient({
-			url : this._getUrl(this.connection),
-			msgClass : AuthGroup.classById(1),
-			params : {
-				clientId : this.clientId,
-				ticket : this.ticket,
-				feedScope : this.sConf.feedScope
-			}
-		}, KeepAliveMessageGroup);
-		this.client.attachListener(this);
-		this.client.enableKeepAlive();
-		this.client.registerGroup(AuthGroup, this);
-		this.client.registerGroup(StreamRegistryMessageGroup, this);
-		
-		this.client.connect();
-	},
-	
-	reconnect : function() {
-		
-		console.log("[DataNode] Id ", this.id, " counter ", this.counter, " is going to re-connect. Current state ", this.state);
-		
-		// update the parameters to be updated when reconnect
-		// the ticket might have changed!!
-		this.client.config.params = {
-			clientId : this.clientId,
-			ticket : this.ticket,
-			feedScope : this.sConf.feedScope
-		}
-		this.client.connect();
-	},
-	
-	_getUrl : function (connection) {
-		return connection.protocol+"://"+connection.hostname+((connection.port !== undefined)?(":"+connection.port):"")+((connection.path !== undefined)?connection.path:"");
-	},
-	
-	/**
-	 * check the consume queue and do registration one by one
-	 */
-	consumeTask : function () {
-		
-		var self = this;
-		
-		if (this.state == DataNode.State.CONNECTED) {
-			var streamByGroup = _.groupBy(this.toBeConsumed, function(streamInfo) { return  streamInfo.group; } );
-			
-			// javascript is single threaded so no need to worry about synchronization here!!
-			this.toBeConsumed = [];			// clear the toBeConsumed list.
-			this.consumeTaskTimeoutID = 0;	// reset the task handler
-			
-			// for each group to be consumed, create a promise, send a message
-			// the promise will be resolved by the onMessage method, or when disconnection
-			_.each(streamByGroup, function(streams, group) {
-				
-				// if no one is removing the stream, then we can consume it
-				// this is to avoid double consumption from two different node
-				// because if removal can only be the result of failing over
-				// or switching to a better source
-				// if the stream is being in the process of consuming, we don't
-				// resend registration as well
-				var canConsume = _.filter(streams, function(streamInfo) {
-					var sid = Util._streamId(streamInfo);
-					
-					return self.removalPromises[sid] === undefined && 
-							self.consumptionPromises[sid] === undefined; 
-				});
-				
-				// we have to create one promise for each stream whose registration
-				// is going to be sent, because we handle at stream level
-				// store those promises to the consuming list
-				var promises = _.reduce(canConsume, function(memo, streamInfo) {
-					
-					var sid = Util._streamId(streamInfo);
-					memo[sid] = Util._newPromise();
-					memo[sid].then(function(result) {
-						console.log("Consumption result of ", sid, " : successful = ",result);
-					});
-					return memo;
-					
-				}, self.consumptionPromises);
-				
-				// send the registration messages
-				var streamNames = _.map(canConsume, function(streamInfo) {
-					return streamInfo.stream;
-				});
-				
-				var req = new StreamRegistryRequest(group, streamNames, StreamRegistryRequest.REGISTRATION);
-				// start consumption by first registration message
-				self.client.send(new MessageWrapper(req, StreamRegistryMessageGroup.REQUEST));
-				
-			});
-		}
-		
-	}, 
-	
-	onEvent : function(event) {
-		
-		console.log("[DataNode] Event from the stack client", event);
-		
-		if (event instanceof EPConnectedEvent) {
-			if(this.state === DataNode.State.DISCONNECTED){
-				this._consumeStreamsOnReconnected();
-			}
-			this.setState(DataNode.State.CONNECTED);
-			console.log("[DataNode] Connect / Re-connect to ", this.connection.hostname,":", this.connection.port, "successfully. Consumption will be started!");
-			
-			this.consumeTaskTimeoutID = window.setTimeout(_.bind(this.consumeTask, this), 100);
-			
-		} else if (event instanceof EPDisconnectedEvent || event instanceof EPConnectionError) {
-			if (this.state == DataNode.State.DESTROYED) return;	// do nothing if the data node is destroyed.
-			
-			this.setState(DataNode.State.DISCONNECTED);
-			console.log("Connect / Re-connect to ",this.connection.hostname,":", this.connection.port, " with error. Retry again later.");
-			
-			// disconnection will result in all consumptionPromises resolve to false, and all 
-			// de-registration promises resolve to true
-			_.each(this.consumptionPromises, function(p, key) { p.resolve(false); } );
-			_.each(this.removalPromises, function(p, key) { p.resolve(true); } );
-			
-			delete this.consumptionPromises;
-			delete this.removalPromises;
-			
-			this.consumptionPromises = {};
-			this.removalPromises = {};
-			
-			if (this.state != DataNode.State.DESTROYED) window.setTimeout(_.bind(this.reconnect, this), this.sConf.reconnectionInterval);
-			else console.log("Data node ", this.id, " has been destroyed. Stop reconnecting.");
-			
-			if (this.consumeTaskTimeoutID !== 0) window.clearTimeout(this.consumeTaskTimeoutID);
-			
-			if (event instanceof EPConnectionError) this.eventBus.emit(DataNode.Event.RECONNECTION_ERROR, this, ++this.numFailedReconnection);
-			else this.numFailedReconnection = 0;
-			
-		}
-		
-	},
-	
-	process : function(messageWrapper) {
-		
-		if (messageWrapper.message instanceof StreamRegistryResponse) {
-			var rep = messageWrapper.message;
-			
-			if (rep.responseType == StreamRegistryResponse.REGISTRATION) {	// registration response
-				
-				this._processRegistrationResult(rep);
-
-			} else if (rep.responseType == StreamRegistryResponse.CONSUMPTION) {
-				
-				this._processConsumptionResult(rep);
-				
-			} else if (rep.responseType == StreamRegistryResponse.DEREGISTRATION) {
-				
-				this._processDeregistrationResult(rep);
-				
-			} else if (rep.responseType == StreamRegistryResponse.STOP_CONSUMPTION) {
-				this._processStopConsumptionResult(rep);
-			}
-		}
-		
-	},
-	
-	_resolveConsumptionPromise : function(group, stream, value) {
-		var id = Util._streamIdFromNames(group, stream);
-		if (this.consumptionPromises[id] !== undefined)
-			this.consumptionPromises[id].resolve(value);
-		
-		delete this.consumptionPromises[id];
-	},
-	
-	_resolveRemovalPromise : function(group, stream, value) {
-		var id = Util._streamIdFromNames(group, stream);
-		if (this.removalPromises[id] !== undefined) {
-			this.removalPromises[id].resolve(value);
-			delete this.removalPromises[id];
-		}
-		
-		delete this.consumptionPromises[id];
-	},
-	
-	_processDeregistrationResult : function (rep) {
-		var self = this;
-		
-		var group = rep.messageGroup;
-		var errorConsumedStreamByGroup = {}; // contain all unsuccessfully consumed streams
-		_.each(rep.streams, function(result, stream) {
-			if (result == StreamRegistryResponse.FAILED) {
-				self._resolveRemovalPromise(group, stream, false);
-			} else {
-				self._resolveRemovalPromise(group, stream, true);
-				if(errorConsumedStreamByGroup[group] === undefined){
-					errorConsumedStreamByGroup[group] = [];
-				}
-				errorConsumedStreamByGroup[group].push(stream);
-			}
-		});
-		if(_.isEmpty(errorConsumedStreamByGroup) == false){
-			console.log("[DataNode] Emitting consumption error after Deregistration")
-			this.eventBus.emit(DataNode.Event.CONSUMPTION_ERROR, this, errorConsumedStreamByGroup);
-		}
-	},
-	
-	_processStopConsumptionResult : function(rep) {
-		var self = this;
-		var group = rep.messageGroup;		
-		var stoppedStreamsByGroup = {};
-		_.each(rep.streams, function(result, stream) {
-			console.log("Stopping stream " + stream);
-			self.removeStream(group, stream, false);
-			if(stoppedStreamsByGroup[group] === undefined){
-				stoppedStreamsByGroup[group] = [];
-			}
-			stoppedStreamsByGroup[group].push(stream); 
-		});
-		if (!_.isEmpty(stoppedStreamsByGroup)) {
-			this.eventBus.emit(DataNode.Event.CONSUMPTION_STOP, this, stoppedStreamsByGroup);
-		}
-	},
-
-	_processConsumptionResult : function (rep) {
-		var self = this;
-		
-		var group = rep.messageGroup;
-		var consumedStreamByGroup = {}; // contain all successfully consumed streams
-		var errorConsumedStreamByGroup = {}; // contain all unsuccessfully consumed streams
-		
-		_.each(rep.streams, function(result, stream) {
-			if (result == StreamRegistryResponse.FAILED) {
-				self._resolveConsumptionPromise(group, stream, false);
-				self.removeStream(group, stream, true);
-				
-				if(errorConsumedStreamByGroup[group] === undefined){
-					errorConsumedStreamByGroup[group] = [];
-				}
-				
-				errorConsumedStreamByGroup[group].push(stream);
-			} else {
-				self._resolveConsumptionPromise(group, stream, true);
-				if(consumedStreamByGroup[group] === undefined){
-					consumedStreamByGroup[group] = [];
-				}
-				consumedStreamByGroup[group].push(stream); 
-			}
-		});
-		
-		if(_.isEmpty(consumedStreamByGroup) == false){
-			this.eventBus.emit(DataNode.Event.CONSUMPTION_START, this, consumedStreamByGroup);
-		}
-		if(_.isEmpty(errorConsumedStreamByGroup) == false){
-			this.eventBus.emit(DataNode.Event.CONSUMPTION_ERROR, this, errorConsumedStreamByGroup);
-		}
-
-	},
-	
-	_processRegistrationResult : function(rep) {
-		
-		var self = this;
-		var group = rep.messageGroup;
-		
-		// complete and remove any failed promise
-		var cannotConsume = _.pairs(rep.streams)
-				.filter(function(p) { return p[1] == StreamRegistryResponse.FAILED; })
-				.map(_.first);
-		
-		// remove these streams from the node
-		cannotConsume.forEach(function(streamName) {
-			self._resolveConsumptionPromise(group, streamName, false);
-			self.removeStream(group, streamName, false);	// the registration is failed, so no need to send deregistration
-		});
-				
-		var canConsume = _.pairs(rep.streams)
-				.filter(function(p) { return p[1] == StreamRegistryResponse.SUCCESS; })
-				.map(_.first)	// get the stream name
-				.filter(function(streamName) {
-					if (self.sConf.processors[group] !== undefined) {	// can consume because processor for the group is defined
-						return true;
-					} else {	// cannot consume because processor is undefined
-						self._resolveConsumptionPromise(group, streamName, false);
-						return false;
-					}
-				});
-		
-		// send consumption message, at this point, the server can be sure that the processor is ready to handle the stream
-		if (!_.isEmpty(canConsume)) {
-			
-			var processor = self.sConf.processors[group];
-			self.client.registerGroup(processor.group(), processor);	// attach processor for all the consumable group, whose processor is defined.
-			
-			var req = new StreamRegistryRequest(group, canConsume, StreamRegistryRequest.CONSUMPTION);
-			self.client.send(new MessageWrapper(req, StreamRegistryMessageGroup.REQUEST));
-		}
-		
-		if (!_.isEmpty(cannotConsume)) {
-			var eventData = {};
-			eventData[group] = cannotConsume;
-			console.log("Emitting consumption error due to registration result failed.", eventData);
-			
-			this.eventBus.emit(DataNode.Event.CONSUMPTION_ERROR, this, eventData);
-		}
-	},
-	
-	setState : function(state) {
-		
-		if (state != this.state) {
-			var oldState = this.state;
-			this.state = state;
-			this.eventBus.emit(DataNode.Event.STATE, this, oldState);
-		}
-		
-	},
-	
-	renewTicket : function(ticket) {
-		this.ticket = ticket;
-		var wrapper = new MessageWrapper(new TicketRenew(this.clientId, this.ticket), AuthGroup.TICKET_RENEW);
-		if (this.state == DataNode.State.CONNECTED) this.client.send(wrapper);
-	},
-	
-	destroy : function() {
-		this.setState(DataNode.State.DESTROYED);
-		console.log("[DataNode] Disconnect the destroyed data node.");
-		this.client.disconnect();
-	}
-	
-});
-
-function Subscriber(configuration) {
-	IEndPointEventSource.call(this);			// initial IEndPointEventSource
-	this.conf = new SubscriberConfig();
-	
-	_.extend(this.conf, configuration);		// copy the configuration over
-	
-	this.consuming = {};		// a map from group / stream to the wrapper
-	this.nodes = {};			// a map from source id to the information related to the source, e.g. state whether connecting, reconnecting, connected or the client used to connect to the node
-	
-	// store the currently available sources retrieved from the discoverer
-	// this data will be used for fast failover whenever there is some 
-	// connection error in a DataNode. 
-	this.currentAvailableSources = null;
-	// Check currentAvailableSources has used.
-	this.hasUsedCurrentSources = false;
-	
-	this.clientId = this.uuid();
-	this.ticket = null;
-	
-	this.eventBus = new EventEmitter();
-	
-	this.eventBus.on(DataNode.Event.RECONNECTION_ERROR, _.bind(this.onNodeReconnectionError, this));
-	this.eventBus.on(DataNode.Event.STATE, _.bind(this.onNodeStateChanged, this));
-	this.eventBus.on(DataNode.Event.CONSUMPTION_START, _.bind(this.onStreamConsumption,this));
-	this.eventBus.on(DataNode.Event.CONSUMPTION_ERROR, _.bind(this.onStreamError, this));
-	this.eventBus.on(DataNode.Event.CONSUMPTION_STOP, _.bind(this.onStreamStop, this));
-}
-function StreamConsumptionWrapper(state, node) {
-	this.state = state;
-	this.node = node;
-}
-
-/**
- * Define Subscriber event and will be fired everytime a client is connected or disconnected.
- */
-function ConsumptionStart(client, streams) {
-	EPEvent.call(this, client._ep);
-	this.client = client;
-	this.streams = streams;
-}
-function ConsumptionStop(client, streams) {
-	EPEvent.call(this, client._ep);
-	this.client = client;
-	this.streams = streams;
-}
-function ConsumptionError(client, streams) {
-	EPEvent.call(this, client._ep);
-	this.client = client;
-	this.streams = streams;
-}
-_c(ConsumptionStart, {}, EPEvent);
-_c(ConsumptionStop, {}, EPEvent);
-_c(ConsumptionError, {}, EPEvent);
-
-Subscriber.Event = {
-	ConsumptionStart : ConsumptionStart,
-	ConsumptionStop : ConsumptionStop,
-	ConsumptionError : ConsumptionError
-};
-
-_c(Subscriber, {
-	
-	consumptionState : {
-		PENDING : 1,		// a stream is in this state when it is waiting for a data node to connect and then perform registration
-		ERROR : 3			// a stream is in this state when its data node is having connection problem
-	},
-	
-	uuid : function (){
- 	
-		return generateUUID();
-
-	},
-	
-	config : function() { return this.conf; },
-	/**
-	 * This method starts the discovery process
-	 */
-	start : function() {
-		var discoverer = new Discoverer(this.config().streamFinder);	// create a discoverer to connect to stream finder
-		if (this.conf.discoverByScope) {
-			discoverer = new Discoverer(this.config().streamFinder, undefined, this.conf.feedScope);	// discover by streamfinder scope
-		}
-		
-		// configure discovery
-		_.each(this.config().discoveryPattern, function(patternList, group) {
-			
-			_.each(patternList, function(pattern) {
-				discoverer.discover(group, pattern);
-			});
-			
-		});
-		
-		_.each(this.config().schemes, function(scheme) {
-			discoverer.using(scheme);
-		});
-		
-		discoverer.on("update", _.bind(this._updateSources, this));
-		discoverer.on("error", _.bind(this._discovererError, this));
-		setInterval(_.bind(this._reconnectStreams, this), 5000);
-		this._makeTicket(true);
-
-		this.discoverer = discoverer;
-	},
-	
-	_makeTicket : function(isStarting) {
-		SessionFactory.makeTicket(this.config().streamFinder, this.clientId, _.bind(function(ticket, error) {
-			if (ticket.length != 0) {
-				
-				this.ticket = ticket;
-				console.log("New ticket obtained "+this.ticket);
-				
-				if (isStarting) {				// if this is not renewal, we should start the discovery process
-					this.discoverer.start();
-				} else {
-					_.each(this.nodes, _.bind(function(node, id) {
-						window.setTimeout(_.bind(node.renewTicket, node, this.ticket), 0);	// asynchronously
-					}, this));
-				}
-				
-				window.setTimeout(_.bind(this._makeTicket, this), this.config().ticketRenewalInterval);
-				
-			} else if (error.length != 0) {
-				window.setTimeout(_.bind(this._makeTicket, this), 1000);
-			}
-		}, this));
-	},
-	
-	_updateSources : function(sources) {
-		
-		console.log("Updating Sources!");
-		console.log(sources);
-		
-		this.oldSources = this.currentAvailableSources;
-		this.currentAvailableSources = sources.streamConnections;
-		this.hasUsedCurrentSources = false;
-		
-		var toBeConnected = {};	// a map of new nodes to be connected
-		var toBeUpdated = {};	// a map of existing node to be updated with new group, streams
-		
-		if (this.oldSources && this.oldSources.length > 0) {
-			var streamIds = [];
-			var nodeIds = [];
-			
-			_.each(sources.streamConnections, function(source) {
-				streamIds.push(Util._streamId(source));
-				nodeIds.push(Util._nodeId(source));
-			});
-			
-			// remove from corresponding old sources not exist in new sources from data nodes
-			var stopStreamsByGroup = {};
-			_.each(this.oldSources, _.bind(function(oldSource) {
-				
-				if (!_.contains(streamIds, Util._streamId(oldSource)) || !_.contains(nodeIds, Util._nodeId(oldSource))) {
-					
-					console.log("[Subscriber] ", oldSource, " is no longer in the discovery results. Going to be removed.")
-					
-					_.each(this.nodes, _.bind(function(dataNode) {
-						
-						if (dataNode.hasStream(oldSource.group, oldSource.stream)) {
-							console.log("[Subscriber] Remove old stream ", oldSource, " from Data Node ", dataNode.id, " dataNode counter ", dataNode.counter);
-							dataNode.removeStream(oldSource.group, oldSource.stream, true);
-							if(stopStreamsByGroup[oldSource.group] === undefined){
-								stopStreamsByGroup[oldSource.group] = [];
-							}
-							stopStreamsByGroup[oldSource.group].push(oldSource.stream);
-						}
-						if(!_.isEmpty(stopStreamsByGroup)) {
-							console.log("[DataNode] Emitting consumption error after removing streams")
-							this.eventBus.emit(DataNode.Event.CONSUMPTION_ERROR, dataNode, stopStreamsByGroup);
-						}
-					}, this));
-				}
-				
-			}, this));
-			
-		}
-		
-		_.each(sources.streamConnections, _.bind(function(source) {	// for each source, we check if we are already consuming it
-			
-			/**
-			 * if we are not currently consuming a particular group & stream,
-			 * or if we are consuming it but the connection for that group & stream is having
-			 * problem, we will gather the group & stream by source id. 
-			 */
-			if (!this._exist(source) || 
-					// if there is error at the node currently consuming the stream
-					// and the connection returned by discoverer is different from the one
-					// currently use (which is causing error)
-					this._error(source) || this._hasUpdatedConnection(source)) {
-				
-				var nodeId = Util._nodeId(source);
-				
-				if (this._hasNode(nodeId)) {
-					if (toBeUpdated[nodeId] == undefined) toBeUpdated[nodeId] = [];
-					toBeUpdated[nodeId].push(source);
-				} else {
-					if (toBeConnected[nodeId] == undefined) toBeConnected[nodeId] = [];
-					toBeConnected[nodeId].push(source);
-				}
-			}
-			
-		}, this));
-		
-		// any group, and stream which are in these list should be removed from current
-		// data node first, before being added to new data node
-		this._connectUpdate(toBeConnected);
-		this._connectUpdate(toBeUpdated);
-		
-	},
-
-	_reconnectStreams : function(sources) {
-		var toBeConnected = {};	// a map of new nodes to be connected
-		var toBeUpdated = {};	// a map of existing node to be updated with new group, streams
-		_.each(this.discoverer.getSources().streamConnections, _.bind(function(source) {	// for each source, we check if we are already consuming it
-
-			/**
-			* if we are not currently consuming a particular group & stream,
-			* or if we are consuming it but the connection for that group & stream is having
-			* problem, we will gather the group & stream by source id. 
-			*/
-			if (!this._exist(source) || 
-				// if there is error at the node currently consuming the stream
-				// and the connection returned by discoverer is different from the one
-				// currently use (which is causing error)
-				(this._error(source) && this._hasUpdatedConnection(source))) {
-
-				console.log("Reconnecting to stopped stream ", source);
-				var nodeId = Util._nodeId(source);
-				if (this._hasNode(nodeId)) {
-					if (toBeUpdated[nodeId] == undefined) toBeUpdated[nodeId] = [];
-					toBeUpdated[nodeId].push(source);
-				} else {
-					if (toBeConnected[nodeId] == undefined) toBeConnected[nodeId] = [];
-					toBeConnected[nodeId].push(source);
-				}
-			}
-
-		}, this));
-		this._connectUpdate(toBeConnected);
-		this._connectUpdate(toBeUpdated);
-	},
-	
-	onNodeReconnectionError : function(dataNode, num) {
-		
-		if (num == this.config().exclusionThreshold) {
-			this.discoverer.excluding(dataNode.id);
-			this._updateSourcesOnNodeReconnectionError();
-			console.log("Excluding node ", dataNode.id, " from discovery");
-		}
-		
-	},
-	
-	_updateSourcesOnNodeReconnectionError : function(){
-		// Check if currentAvailableSources is not empty and not used.
-		if(this.hasUsedCurrentSource == false && _.isEmpty(this.currentAvailableSources) == false){
-			this._updateSources(this.currentAvailableSources);
-			this.hasUsedCurrentSources = true; // mark currentAvailableSources has used.
-		}
-	},
-	
-	onNodeStateChanged : function(dataNode, oldState) {
-		
-		// if the data node is destroyed or disconnected. Need to inform clients about the consumption error
-		// also mark those sources provided by that nodes as having error state
-		if (dataNode.state == DataNode.State.DESTROYED || dataNode.state == DataNode.State.DISCONNECTED) {
-							
-			for(var property in this.consuming){
-				if(this.consuming.hasOwnProperty(property)){
-					if(this.consuming[property].node.id == dataNode.id){
-						this.consuming[property].state = this.consumptionState.ERROR;
-					}
-				}
-			}
-			
-			var streamsByGroup = {};
-			
-			for(var property in dataNode.streams){
-				if(dataNode.streams.hasOwnProperty(property)){
-					streamsByGroup[property] = [];
-					for (streamInfo in dataNode.streams[property])
-						streamsByGroup[property].push(streamInfo.stream);
-				}
-			}
-			
-			if (!_.isEmpty(streamsByGroup)) {	// if streamsByGroup doesn't contain any thing, no need to emit event
-				console.log("[Subscriber] emitting consumption error due to data node failed.", dataNode.state);
-				this.dispatch(new Subscriber.Event.ConsumptionError(dataNode.client, streamsByGroup));
-			}
-		}
-		
-		if (oldState === DataNode.State.DISCONNECTED && dataNode.state === DataNode.State.CONNECTED) {
-			this.discoverer.removeExclusion(dataNode.id);
-			console.log("Node ", dataNode.id, " resume successfully. Remove it from exclusion.");
-		} else if (dataNode.state === DataNode.State.DESTROYED) {
-			// when it's destroyed, we are not going to reconnect to it automatically, so remove from exclusion
-			this.discoverer.removeExclusion(dataNode.id);
-			delete this.nodes[dataNode.id];
-			console.log("Node ", dataNode.id, " is destroyed and removed from subscriber!");
-			
-		}
-		
-	},
-
-	onStreamConsumption : function(dataNode, streamsByGroup){
-		this.dispatch(new Subscriber.Event.ConsumptionStart(dataNode.client, streamsByGroup));
-	},
-	
-	onStreamError : function(dataNode, streamsByGroup){
-		this.dispatch(new Subscriber.Event.ConsumptionError(dataNode.client, streamsByGroup));
-	},
-
-	onStreamStop : function(dataNode, streamsByGroup){
-		this.dispatch(new Subscriber.Event.ConsumptionStop(dataNode.client, streamsByGroup));
-	},
-	
-	_discovererError : function (err) {
-		
-	},
-	
-	/**
-	 * This method remove the group & stream which are going to be 
-	 * consumed in NEW DataNode or to be ADDED to other EXISTING
-	 * DataNodes from the current DataNodes they belong to (because current DataNodes
-	 * might be having connection error, which marks the respective group & streams
-	 * as error). Only when there is no one else consuming the stream, 
-	 * the stream can be registered into a new node. 
-	 * 
-	 * @param sources : a map from the nodeId of nodes to be added / updated to the group and stream. 
-	 */
-	_connectUpdate : function(sources) {
-		
-		_.each(sources, _.bind(function(connections, nodeId) {
-						
-			_.each(connections, _.bind(function(streamInfo) {
-				var streamId = Util._streamId(streamInfo);
-				
-				if (!this.consuming[streamId]) this.consuming[streamId] = new StreamConsumptionWrapper();
-				
-				var wrapper = this.consuming[streamId];
-				
-				var promise = Promise.resolve(0);
-				
-				// this will automatically destroy the node if there is no stream for it to consume
-				if (wrapper !== undefined && wrapper.node != undefined && wrapper.state == this.consumptionState.ERROR)
-					promise = wrapper.node.removeStream(streamInfo.group, streamInfo.stream, false);	// if consumption state is already error, no need to send deregistration
-				
-				promise.done = function(onFulfilled, onRejected) {
-					this.then(onFulfilled, onRejected)
-						.catch(function (reason) {
-							console.log("Promise Rejected: " + reason + "\n" + reason.stack);
-						});
-				};
-				
-				// this stream is going to be consume soon
-				wrapper.state = this.consumptionState.PENDING;
-				/**
-				 * when the promise is done
-				 * it means that there is no one consuming the group & stream
-				 * it's okay now that I'll start consuming it
-				 */
-				if (this.nodes[nodeId]) {	// if the node already existed
-					promise.done(_.bind(function(result) { 
-						
-						this.nodes[nodeId].consumeStream(streamInfo);
-						wrapper.node = this.nodes[nodeId];	// mark this so that no one else can consume this stream
-						
-					}, this));
-				} else {	// a new data node has to be created
-					
-					this.nodes[nodeId] = new DataNode(nodeId, this.clientId, this.ticket, streamInfo, this.eventBus, this.config());
-					promise.done(_.bind(function(result) {
-						wrapper.node = this.nodes[nodeId];	// mark this so that no one else can consume this stream
-						this.nodes[nodeId].connect();
-						
-					}, this));
-					
-				}
-				
-			}, this));
-			
-		}, this));
-		
-	},
-	
-	/**
-	 * Check if a group and stream existed
-	 * This means, the subscriber is consuming data for this group and stream from
-	 * a source.
-	 */
-	_exist : function(source) {
-		var streamId = Util._streamId(source);
-		
-		if (this.consuming[streamId]) return true;
-		
-		return false;
-	},
-	
-	/**
-	 * Check if a consuming group and stream is having any problem.
-	 * A consuming group and stream has problem if the DataNode for it is having 
-	 * connection problem.
-	 */
-	_error : function(source) {
-		var streamId = Util._streamId(source);
-		
-		if (this.consuming[streamId] && this.consuming[streamId].state == this.consumptionState.ERROR) return true;
-		
-		return false;
-	},
-	
-	/**
-	 * Check if a source returned by discoverer has newer connection
-	 * than the one being used
-	 */
-	_hasUpdatedConnection : function (source) {
-		var streamId = Util._streamId(source);
-		
-		if (!this.consuming[streamId] || !this.consuming[streamId].node || 
-				this.consuming[streamId].node.state == DataNode.State.DESTROYED) {
-			console.log("Has updated source");
-			
-			if (!this.consuming[streamId]) 
-				console.log("Because the source is not consuming");
-			else if (!this.consuming[streamId].node) 
-				console.log("Because the source is not consumed from a node");
-			else if (this.consuming[streamId].node == DataNode.State.DESTROYED) 
-				console.log("Because the source's node does not exist");
-			
-			return true;	// if this source is not in the consuming list or if it is, but there is no node for it
-		} else if (Util._nodeId(source) != this.consuming[streamId].node.id) {
-			console.log("Has updated source because the new node id is different from previous!");
-			return true;	// if the service id of the source is different from service id of the node
-		}
-		
-		return false;
-	},
-	
-	_hasNode : function(nodeId) {
-		return this.nodes[nodeId] !== undefined;
-	},
-	
-}, IEndPointEventSource);
-
-module.exports = exports = {
-	Subscriber : Subscriber,
-	SubscriberConfig : SubscriberConfig
-}
+var coreMsges = require("./CoreMessages")
+var deliveryStack = require("./DeliveryStack")
+var _ = require("underscore")
+var EventEmitter = require("eventemitter3");
+
+var streamFinder = require("@jayesol/jayeson.lib.streamfinder");
+var Discoverer = streamFinder.Discoverer;
+
+var util = require("./util")
+var _c = util._c
+
+var DefaultClient = deliveryStack.DefaultClient
+var EPEvent = deliveryStack.EPEvent
+var EPConnectedEvent = deliveryStack.EPConnectedEvent
+var EPConnectionError = deliveryStack.EPConnectionError
+var EPDisconnectedEvent = deliveryStack.EPDisconnectedEvent
+var IEndPointEventSource = deliveryStack.IEndPointEventSource
+var generateUUID = deliveryStack.generateUUID;
+var Util = deliveryStack.Util;
+var MessageWrapper = deliveryStack.MessageWrapper;
+
+var StreamRegistryMessageGroup = coreMsges.StreamRegistryMessageGroup;
+var	EmptyMessageClass = coreMsges.EmptyMessageClass;
+var	StringMessageClass = coreMsges.StringMessageClass;
+var	JSonMessageClass = coreMsges.JSonMessageClass;
+var	StreamRegistryRequest = coreMsges.StreamRegistryRequest;
+var	StreamRegistryResponse = coreMsges.StreamRegistryResponse;
+var	AuthGroup = coreMsges.AuthGroup;
+var	GenericMessageGroup = coreMsges.GenericMessageGroup;
+var	KeepAliveMessageGroup = coreMsges.KeepAliveMessageGroup;
+var TicketRenew = coreMsges.TicketRenew;
+
+/**
+ * @author Ryan
+ * 
+ * This depends on external libraries: underscorejs, eventemitter, promisejs
+ * 
+ * This depends on internal libraries: coremessage, deliverystack, jsdiscoverer
+ * 
+ * The subscriber has the following features:
+ * TODO: FILL IN THE FEATURES 
+ * 
+ */
+
+/**
+ * Represents a configuration being passed
+ * into the subscriber. In the javascript context,
+ * 
+ * The discovery pattern has the following format
+ * {
+ * 		id1 : [pattern 1, pattern 2 .... ],
+ * 		id2 : [pattern A, pattern B .... ]			
+ * }
+ * 
+ * example
+ * 
+ * {
+ * 		8 : ["*_LIVE", "SBO_*"],	// price feed
+ * 		7 : ["order_ryan"]			// order data feed for user ryan
+ * }
+ * 
+ * The processor setup is as following:
+ * {
+ * 		id1 : processor 1 of type IMessageGroupProcessor,
+ * 		id2 : processor 2 of type IMessageGroupProcessor
+ * 
+ * }
+ * 
+ */
+function SubscriberConfig() {
+	this.autoDiscover = true;	// by default, enable auto discovery
+	this.streamFinder = "";		// the URL to connect to the stream finder
+	this.username = "";			// if we want to login directly to the stream finder
+	this.password = "";
+	this.discoveryPattern = {};
+	this.processors = {};
+	this.schemes = ["ws"];		// by default we consume from ws
+	this.ticketRenewalInterval = 14*60*1000;	// default to 14 minutes
+	this.reconnectionInterval = 3000;			// reconnection to a relay node
+	this.exclusionThreshold = 3;				// the number of reconnection failed before the relay node is added to the excluded list
+	this.feedScope = "";
+	this.discoverByScope = true; //new version will discover by scope
+}
+
+/**
+ * Represents a data node. This encapsulate the following information
+ * <li>The client used to connect to the node</li>
+ * <li>The stream being consumed from the node</li>
+ * 
+ * It also provides functionalities such as reconnection. It listen to ticket being renewed
+ * by the subscriber and send the ticket to the server accordingly.
+ * 
+ * @params
+ * 	<li>id - id of the data node</li>
+ * 	<li>streamInfo - the first stream information used for connection<br/>
+ *	<pre> 
+ * 	{
+ *		"group": "<groupId>",
+ *		"stream": "<streamName>",
+ *		"level": <level>,
+ *		"serviceId" : <serviceId>,
+ *		"connection": {
+ *   		"protocol": "<protocol>",
+ *   		"hostname": "<hostname>",
+ *   		"port": <port>
+ * 		}
+ *	}
+ *	</pre>
+ *	</li>
+ *	<li>clientId - id of the client</li>
+ *	<li>ticket - ticket of the subscriber</li>
+ *	<li>eventBus - an Event Emitter which is responsible to propagate events from DataNode</li>
+ *	<li>subscriberConfig - configuration of the subscriber that created this data node</li>
+ */
+var DATA_NODE_COUNTER = 0;
+
+function DataNode(id, clientId, ticket, streamInfo, eventBus, subscriberConfig) {
+	this.client = null;
+	/**
+	 * This data structure has the following form:
+	 * { group -> [stream information] }
+	 */
+	this.streams = {};
+	this.id = id;		// id of the data node
+	
+	this.counter = DATA_NODE_COUNTER++;
+	console.log("[DataNode] Node ", id, " counter ", this.counter, " is created!");
+	
+	/**
+	 * Current state of the data node
+	 */
+	this.state = DataNode.State.CREATED;
+	this.connection = streamInfo.connection;
+	
+	this.toBeConsumed = [];
+	
+	this.consumeStream(streamInfo);
+	
+	this.clientId = clientId;
+	this.ticket = ticket;
+	
+	this.consumeTaskTimeoutID = 0;
+	
+	this.eventBus = eventBus;
+	
+	this.sConf = subscriberConfig;
+	
+	// the list of promises which will be resolved when stop consumption messages
+	// for a stream is handled and replied by the server. The promise
+	// shall be successful if there is disconnection while waiting for the reply
+	// Format: streamId -> promiseObject e.g. "8_CROWN"
+	// the promise value is true / false
+	this.removalPromises = {};
+	// the list of promises which will be resolved when consumption messages
+	// for a stream is handled and replied by the server. The promise shall
+	// resolve to failed if there is disconnection waiting for the reply. The promise
+	// is created when the registration (not consumption) for the stream starts.
+	// The promise value is true / false.
+	this.consumptionPromises = {};
+	
+};
+DataNode.State = {
+	CREATED : 1,
+	CONNECTING : 2,
+	CONNECTED : 3,
+	RECONNECTING : 4,
+	DISCONNECTED : 5,
+	DESTROYED : 6
+};
+DataNode.Event = {
+	STATE : "DN_STATE",
+	RECONNECTION_ERROR : "DN_RE_ERROR",
+	CONSUMPTION_ERROR : "DN_CONSUMPTION_ERROR",
+	CONSUMPTION_START : "DN_CONSUMPTION_START",
+	CONSUMPTION_STOP : "DN_CONSUMPTION_STOP",
+};
+_c(DataNode, {
+	
+	/**
+	 * This method return a PROMISE of the removal of the stream from the 
+	 * from consumption. This has to be asynchronous because involving sending
+	 * deregistration message to the stream registry of the consuming node. We however, only
+	 * send unregister when the node is in CONNECTED STATE and the person invoke this method
+	 * specify he want to do so (sendDeregistration = true)
+	 * 
+	 * There are two cases that a stream can be removed from a data node:
+	 * <li>The node that consuming the stream has an error and needs reconnection</li>
+	 * <li>A better node can be found for the stream according to the stream selection strategy</li>
+	 * 
+	 * @return a promise which is resolved when the stream is completely removed from consumption.
+	 */
+	removeStream : function(group, stream, sendDeregistration) {
+		
+		var id = Util._streamIdFromNames(group, stream);
+		
+		var deregistrationAction = _.bind(function(registrationResult) {
+			// the unregistration promise. This is done when the
+			// the stream is unregister from this data node by sending message to
+			// the data source
+			var promise;
+			
+			// the action to be done, remove the stream from the array
+			var unregisterPostAction = _.bind(function(result) {
+				if (result && this.streams[group]) {
+					var index;
+					
+					for (index = 0; index < this.streams[group].length; index++)
+						if (stream == this.streams[group][index].stream) break;
+					
+					if (index < this.streams[group].length) this.streams[group].splice(index, 1);
+					
+					if (this.streams[group].length == 0) delete this.streams[group];
+					
+					// if there is no more stream this node is consuming, just destroy itself
+					if (_.isEmpty(this.streams)) this.destroy();
+				}
+				
+				return result;
+			}, this);
+			
+			if (this.state == DataNode.State.CONNECTED && sendDeregistration) {
+				
+				
+				promise = Util._newPromise();
+				
+				
+				this.removalPromises[id] = promise;
+				
+				var req = new StreamRegistryRequest(group, [stream], StreamRegistryRequest.DEREGISTRATION);
+				this.client.send(new MessageWrapper(req, StreamRegistryMessageGroup.REQUEST));
+
+				var self = this;
+				window.setTimeout(function() {
+					//proceed with deregistration if no deregistration responses are received.
+					if (self.removalPromises[id] !== undefined) {
+						console.warn("Timeout waiting for deregistration response, proceeding to treat stream " + group + ":" + stream + " as deregistered for " + self.connection.hostname + ":" + self.connection.port);
+						var streamResponse = {};
+						streamResponse[stream] = StreamRegistryResponse.DEREGISTRATION_TIMEOUT;
+						var response = new StreamRegistryResponse(group, streamResponse, StreamRegistryResponse.DEREGISTRATION);
+
+						self._processDeregistrationResult(response);
+					}
+				}, 7000);
+				
+				// chain the promise
+				return promise.then(unregisterPostAction);
+				
+			} else {
+				// if the DataNode is not in connected state, no need to send anything!
+				promise = Promise.resolve(unregisterPostAction(true));
+				return promise;
+			}
+		}, this);
+		
+		// if there is a consumption in progress, wait for it! We will de-register the stream after the it finish consumption
+		// de-registration only happens in failing over or switching to better stream, so it's okay to de-register
+		// even when the pending consumption is successful. We switch anyway.
+		if (this.consumptionPromises && this.consumptionPromises[id] !== undefined) {	
+			return this.consumptionPromises[id].then(deregistrationAction);
+		} else {
+			return deregistrationAction(true);
+		}
+
+	},
+	
+	hasStream : function(group, stream) {
+		if (!this.streams[group]) {
+			return false;
+		}
+		var stream = _.each(this.streams[group], function(node) { return node.stream == stream; } );
+		return stream !== undefined;
+	},
+	
+	/**
+	 * Start consumption of a particular stream by putting it into tobeConsumed queue.
+	 * When a stream is consumed successfully, DataNode will emit an event
+	 * so that other parties know about it.
+	 */
+	consumeStream : function(streamInfo) {
+		
+		console.log("Node "+this.id+" is going to consume "+streamInfo.group+"_"+streamInfo.stream);
+		
+		this.toBeConsumed.push(streamInfo);
+		
+		if (!this.streams[streamInfo.group]) this.streams[streamInfo.group] = [];
+		
+		this.streams[streamInfo.group].push(streamInfo);	// put the stream into the list of streams that this DataNode handling
+		
+		if (this.consumeTaskTimeoutID == 0 && this.state == DataNode.State.CONNECTED)
+			this.consumeTaskTimeoutID = window.setTimeout(_.bind(this.consumeTask, this), 100);
+	},
+	
+	_consumeStreamsOnReconnected : function(){
+		
+		if (_.isEmpty(this.streams)) this.destroy();
+		
+		for(var group in this.streams){
+			if(this.streams.hasOwnProperty(group)){
+				for(var i=0 ; i<this.streams[group].length; i++){
+					this.toBeConsumed.push(this.streams[group][i]);
+				}
+			}
+		}
+	},
+	
+	connect : function() {
+		
+		console.log("[DataNode] Id ", this.id, " counter ", this.counter, " is going to connect. Current state ", this.state);
+		
+		//temp code
+		//this.connection.hostname = "local.vodds.jayeson.com.sg";
+		//this.connection.port = 5000;
+
+		this.client = new DefaultClient({
+			url : this._getUrl(this.connection),
+			msgClass : AuthGroup.classById(1),
+			params : {
+				clientId : this.clientId,
+				ticket : this.ticket,
+				feedScope : this.sConf.feedScope
+			}
+		}, KeepAliveMessageGroup);
+		this.client.attachListener(this);
+		this.client.enableKeepAlive();
+		this.client.registerGroup(AuthGroup, this);
+		this.client.registerGroup(StreamRegistryMessageGroup, this);
+		
+		this.client.connect();
+	},
+	
+	reconnect : function() {
+		
+		console.log("[DataNode] Id ", this.id, " counter ", this.counter, " is going to re-connect. Current state ", this.state);
+		
+		// update the parameters to be updated when reconnect
+		// the ticket might have changed!!
+		this.client.config.params = {
+			clientId : this.clientId,
+			ticket : this.ticket,
+			feedScope : this.sConf.feedScope
+		}
+		this.client.connect();
+	},
+	
+	_getUrl : function (connection) {
+		return connection.protocol+"://"+connection.hostname+((connection.port !== undefined)?(":"+connection.port):"")+((connection.path !== undefined)?connection.path:"");
+	},
+	
+	/**
+	 * check the consume queue and do registration one by one
+	 */
+	consumeTask : function () {
+		
+		var self = this;
+		
+		if (this.state == DataNode.State.CONNECTED) {
+			var streamByGroup = _.groupBy(this.toBeConsumed, function(streamInfo) { return  streamInfo.group; } );
+			
+			// javascript is single threaded so no need to worry about synchronization here!!
+			this.toBeConsumed = [];			// clear the toBeConsumed list.
+			this.consumeTaskTimeoutID = 0;	// reset the task handler
+			
+			// for each group to be consumed, create a promise, send a message
+			// the promise will be resolved by the onMessage method, or when disconnection
+			_.each(streamByGroup, function(streams, group) {
+				
+				// if no one is removing the stream, then we can consume it
+				// this is to avoid double consumption from two different node
+				// because if removal can only be the result of failing over
+				// or switching to a better source
+				// if the stream is being in the process of consuming, we don't
+				// resend registration as well
+				var canConsume = _.filter(streams, function(streamInfo) {
+					var sid = Util._streamId(streamInfo);
+					
+					return self.removalPromises[sid] === undefined && 
+							self.consumptionPromises[sid] === undefined; 
+				});
+				
+				// we have to create one promise for each stream whose registration
+				// is going to be sent, because we handle at stream level
+				// store those promises to the consuming list
+				var promises = _.reduce(canConsume, function(memo, streamInfo) {
+					
+					var sid = Util._streamId(streamInfo);
+					memo[sid] = Util._newPromise();
+					memo[sid].then(function(result) {
+						console.log("Consumption result of ", sid, " : successful = ",result);
+					});
+					return memo;
+					
+				}, self.consumptionPromises);
+				
+				// send the registration messages
+				var streamNames = _.map(canConsume, function(streamInfo) {
+					return streamInfo.stream;
+				});
+				
+				var req = new StreamRegistryRequest(group, streamNames, StreamRegistryRequest.REGISTRATION);
+				// start consumption by first registration message
+				self.client.send(new MessageWrapper(req, StreamRegistryMessageGroup.REQUEST));
+				
+			});
+		}
+		
+	}, 
+	
+	onEvent : function(event) {
+		
+		console.log("[DataNode] Event from the stack client", event);
+		
+		if (event instanceof EPConnectedEvent) {
+			if(this.state === DataNode.State.DISCONNECTED){
+				this._consumeStreamsOnReconnected();
+			}
+			this.setState(DataNode.State.CONNECTED);
+			console.log("[DataNode] Connect / Re-connect to ", this.connection.hostname,":", this.connection.port, "successfully. Consumption will be started!");
+			
+			this.consumeTaskTimeoutID = window.setTimeout(_.bind(this.consumeTask, this), 100);
+			
+		} else if (event instanceof EPDisconnectedEvent || event instanceof EPConnectionError) {
+			if (this.state == DataNode.State.DESTROYED) return;	// do nothing if the data node is destroyed.
+			
+			this.setState(DataNode.State.DISCONNECTED);
+			console.log("Connect / Re-connect to ",this.connection.hostname,":", this.connection.port, " with error. Retry again later.");
+			
+			// disconnection will result in all consumptionPromises resolve to false, and all 
+			// de-registration promises resolve to true
+			_.each(this.consumptionPromises, function(p, key) { p.resolve(false); } );
+			_.each(this.removalPromises, function(p, key) { p.resolve(true); } );
+			
+			delete this.consumptionPromises;
+			delete this.removalPromises;
+			
+			this.consumptionPromises = {};
+			this.removalPromises = {};
+			
+			if (this.state != DataNode.State.DESTROYED) window.setTimeout(_.bind(this.reconnect, this), this.sConf.reconnectionInterval);
+			else console.log("Data node ", this.id, " has been destroyed. Stop reconnecting.");
+			
+			if (this.consumeTaskTimeoutID !== 0) window.clearTimeout(this.consumeTaskTimeoutID);
+			
+			if (event instanceof EPConnectionError) this.eventBus.emit(DataNode.Event.RECONNECTION_ERROR, this, ++this.numFailedReconnection);
+			else this.numFailedReconnection = 0;
+			
+		}
+		
+	},
+	
+	process : function(messageWrapper) {
+		
+		if (messageWrapper.message instanceof StreamRegistryResponse) {
+			var rep = messageWrapper.message;
+			
+			if (rep.responseType == StreamRegistryResponse.REGISTRATION) {	// registration response
+				
+				this._processRegistrationResult(rep);
+
+			} else if (rep.responseType == StreamRegistryResponse.CONSUMPTION) {
+				
+				this._processConsumptionResult(rep);
+				
+			} else if (rep.responseType == StreamRegistryResponse.DEREGISTRATION) {
+				
+				this._processDeregistrationResult(rep);
+				
+			} else if (rep.responseType == StreamRegistryResponse.STOP_CONSUMPTION) {
+				this._processStopConsumptionResult(rep);
+			}
+		}
+		
+	},
+	
+	_resolveConsumptionPromise : function(group, stream, value) {
+		var id = Util._streamIdFromNames(group, stream);
+		if (this.consumptionPromises[id] !== undefined)
+			this.consumptionPromises[id].resolve(value);
+		
+		delete this.consumptionPromises[id];
+	},
+	
+	_resolveRemovalPromise : function(group, stream, value) {
+		var id = Util._streamIdFromNames(group, stream);
+		if (this.removalPromises[id] !== undefined) {
+			this.removalPromises[id].resolve(value);
+			delete this.removalPromises[id];
+		}
+		
+		delete this.consumptionPromises[id];
+	},
+	
+	_processDeregistrationResult : function (rep) {
+		var self = this;
+		
+		var group = rep.messageGroup;
+		var errorConsumedStreamByGroup = {}; // contain all unsuccessfully consumed streams
+		_.each(rep.streams, function(result, stream) {
+			if (result == StreamRegistryResponse.FAILED) {
+				self._resolveRemovalPromise(group, stream, false);
+			} else {
+				self._resolveRemovalPromise(group, stream, true);
+				if(errorConsumedStreamByGroup[group] === undefined){
+					errorConsumedStreamByGroup[group] = [];
+				}
+				errorConsumedStreamByGroup[group].push(stream);
+			}
+		});
+		if(_.isEmpty(errorConsumedStreamByGroup) == false){
+			console.log("[DataNode] Emitting consumption error after Deregistration")
+			this.eventBus.emit(DataNode.Event.CONSUMPTION_ERROR, this, errorConsumedStreamByGroup);
+		}
+	},
+	
+	_processStopConsumptionResult : function(rep) {
+		var self = this;
+		var group = rep.messageGroup;		
+		var stoppedStreamsByGroup = {};
+		_.each(rep.streams, function(result, stream) {
+			console.log("Stopping stream " + stream);
+			self.removeStream(group, stream, false);
+			if(stoppedStreamsByGroup[group] === undefined){
+				stoppedStreamsByGroup[group] = [];
+			}
+			stoppedStreamsByGroup[group].push(stream); 
+		});
+		if (!_.isEmpty(stoppedStreamsByGroup)) {
+			this.eventBus.emit(DataNode.Event.CONSUMPTION_STOP, this, stoppedStreamsByGroup);
+		}
+	},
+
+	_processConsumptionResult : function (rep) {
+		var self = this;
+		
+		var group = rep.messageGroup;
+		var consumedStreamByGroup = {}; // contain all successfully consumed streams
+		var errorConsumedStreamByGroup = {}; // contain all unsuccessfully consumed streams
+		
+		_.each(rep.streams, function(result, stream) {
+			if (result == StreamRegistryResponse.FAILED) {
+				self._resolveConsumptionPromise(group, stream, false);
+				self.removeStream(group, stream, true);
+				
+				if(errorConsumedStreamByGroup[group] === undefined){
+					errorConsumedStreamByGroup[group] = [];
+				}
+				
+				errorConsumedStreamByGroup[group].push(stream);
+			} else {
+				self._resolveConsumptionPromise(group, stream, true);
+				if(consumedStreamByGroup[group] === undefined){
+					consumedStreamByGroup[group] = [];
+				}
+				consumedStreamByGroup[group].push(stream); 
+			}
+		});
+		
+		if(_.isEmpty(consumedStreamByGroup) == false){
+			this.eventBus.emit(DataNode.Event.CONSUMPTION_START, this, consumedStreamByGroup);
+		}
+		if(_.isEmpty(errorConsumedStreamByGroup) == false){
+			this.eventBus.emit(DataNode.Event.CONSUMPTION_ERROR, this, errorConsumedStreamByGroup);
+		}
+
+	},
+	
+	_processRegistrationResult : function(rep) {
+		
+		var self = this;
+		var group = rep.messageGroup;
+		
+		// complete and remove any failed promise
+		var cannotConsume = _.pairs(rep.streams)
+				.filter(function(p) { return p[1] == StreamRegistryResponse.FAILED; })
+				.map(_.first);
+		
+		// remove these streams from the node
+		cannotConsume.forEach(function(streamName) {
+			self._resolveConsumptionPromise(group, streamName, false);
+			self.removeStream(group, streamName, false);	// the registration is failed, so no need to send deregistration
+		});
+				
+		var canConsume = _.pairs(rep.streams)
+				.filter(function(p) { return p[1] == StreamRegistryResponse.SUCCESS; })
+				.map(_.first)	// get the stream name
+				.filter(function(streamName) {
+					if (self.sConf.processors[group] !== undefined) {	// can consume because processor for the group is defined
+						return true;
+					} else {	// cannot consume because processor is undefined
+						self._resolveConsumptionPromise(group, streamName, false);
+						return false;
+					}
+				});
+		
+		// send consumption message, at this point, the server can be sure that the processor is ready to handle the stream
+		if (!_.isEmpty(canConsume)) {
+			
+			var processor = self.sConf.processors[group];
+			self.client.registerGroup(processor.group(), processor);	// attach processor for all the consumable group, whose processor is defined.
+			
+			var req = new StreamRegistryRequest(group, canConsume, StreamRegistryRequest.CONSUMPTION);
+			self.client.send(new MessageWrapper(req, StreamRegistryMessageGroup.REQUEST));
+		}
+		
+		if (!_.isEmpty(cannotConsume)) {
+			var eventData = {};
+			eventData[group] = cannotConsume;
+			console.log("Emitting consumption error due to registration result failed.", eventData);
+			
+			this.eventBus.emit(DataNode.Event.CONSUMPTION_ERROR, this, eventData);
+		}
+	},
+	
+	setState : function(state) {
+		
+		if (state != this.state) {
+			var oldState = this.state;
+			this.state = state;
+			this.eventBus.emit(DataNode.Event.STATE, this, oldState);
+		}
+		
+	},
+	
+	renewTicket : function(ticket) {
+		this.ticket = ticket;
+		var wrapper = new MessageWrapper(new TicketRenew(this.clientId, this.ticket), AuthGroup.TICKET_RENEW);
+		if (this.state == DataNode.State.CONNECTED) this.client.send(wrapper);
+	},
+	
+	destroy : function() {
+		this.setState(DataNode.State.DESTROYED);
+		console.log("[DataNode] Disconnect the destroyed data node.");
+		this.client.disconnect();
+	}
+	
+});
+
+function Subscriber(configuration) {
+	IEndPointEventSource.call(this);			// initial IEndPointEventSource
+	this.conf = new SubscriberConfig();
+	
+	_.extend(this.conf, configuration);		// copy the configuration over
+	
+	this.consuming = {};		// a map from group / stream to the wrapper
+	this.nodes = {};			// a map from source id to the information related to the source, e.g. state whether connecting, reconnecting, connected or the client used to connect to the node
+	
+	// store the currently available sources retrieved from the discoverer
+	// this data will be used for fast failover whenever there is some 
+	// connection error in a DataNode. 
+	this.currentAvailableSources = null;
+	// Check currentAvailableSources has used.
+	this.hasUsedCurrentSources = false;
+	
+	this.clientId = this.uuid();
+	this.ticket = null;
+	
+	this.eventBus = new EventEmitter();
+	
+	this.eventBus.on(DataNode.Event.RECONNECTION_ERROR, _.bind(this.onNodeReconnectionError, this));
+	this.eventBus.on(DataNode.Event.STATE, _.bind(this.onNodeStateChanged, this));
+	this.eventBus.on(DataNode.Event.CONSUMPTION_START, _.bind(this.onStreamConsumption,this));
+	this.eventBus.on(DataNode.Event.CONSUMPTION_ERROR, _.bind(this.onStreamError, this));
+	this.eventBus.on(DataNode.Event.CONSUMPTION_STOP, _.bind(this.onStreamStop, this));
+}
+function StreamConsumptionWrapper(state, node) {
+	this.state = state;
+	this.node = node;
+}
+
+/**
+ * Define Subscriber event and will be fired everytime a client is connected or disconnected.
+ */
+function ConsumptionStart(client, streams) {
+	EPEvent.call(this, client._ep);
+	this.client = client;
+	this.streams = streams;
+}
+function ConsumptionStop(client, streams) {
+	EPEvent.call(this, client._ep);
+	this.client = client;
+	this.streams = streams;
+}
+function ConsumptionError(client, streams) {
+	EPEvent.call(this, client._ep);
+	this.client = client;
+	this.streams = streams;
+}
+_c(ConsumptionStart, {}, EPEvent);
+_c(ConsumptionStop, {}, EPEvent);
+_c(ConsumptionError, {}, EPEvent);
+
+Subscriber.Event = {
+	ConsumptionStart : ConsumptionStart,
+	ConsumptionStop : ConsumptionStop,
+	ConsumptionError : ConsumptionError
+};
+
+_c(Subscriber, {
+	
+	consumptionState : {
+		PENDING : 1,		// a stream is in this state when it is waiting for a data node to connect and then perform registration
+		ERROR : 3			// a stream is in this state when its data node is having connection problem
+	},
+	
+	uuid : function (){
+ 	
+		return generateUUID();
+
+	},
+	
+	config : function() { return this.conf; },
+	/**
+	 * This method starts the discovery process
+	 */
+	start : function() {
+		var discoverer = new Discoverer(this.config().streamFinder);	// create a discoverer to connect to stream finder
+		if (this.conf.discoverByScope) {
+			discoverer = new Discoverer(this.config().streamFinder, undefined, this.conf.feedScope);	// discover by streamfinder scope
+		}
+		
+		// configure discovery
+		_.each(this.config().discoveryPattern, function(patternList, group) {
+			
+			_.each(patternList, function(pattern) {
+				discoverer.discover(group, pattern);
+			});
+			
+		});
+		
+		_.each(this.config().schemes, function(scheme) {
+			discoverer.using(scheme);
+		});
+		
+		discoverer.on("update", _.bind(this._updateSources, this));
+		discoverer.on("error", _.bind(this._discovererError, this));
+		setInterval(_.bind(this._reconnectStreams, this), 5000);
+		this._makeTicket(true);
+
+		this.discoverer = discoverer;
+	},
+	
+	_makeTicket : function(isStarting) {
+		SessionFactory.makeTicket(this.config().streamFinder, this.clientId, _.bind(function(ticket, error) {
+			if (ticket.length != 0) {
+				
+				this.ticket = ticket;
+				console.log("New ticket obtained "+this.ticket);
+				
+				if (isStarting) {				// if this is not renewal, we should start the discovery process
+					this.discoverer.start();
+				} else {
+					_.each(this.nodes, _.bind(function(node, id) {
+						window.setTimeout(_.bind(node.renewTicket, node, this.ticket), 0);	// asynchronously
+					}, this));
+				}
+				
+				window.setTimeout(_.bind(this._makeTicket, this), this.config().ticketRenewalInterval);
+				
+			} else if (error.length != 0) {
+				window.setTimeout(_.bind(this._makeTicket, this), 1000);
+			}
+		}, this));
+	},
+	
+	_updateSources : function(sources) {
+		
+		console.log("Updating Sources!");
+		console.log(sources);
+		
+		this.oldSources = this.currentAvailableSources;
+		this.currentAvailableSources = sources.streamConnections;
+		this.hasUsedCurrentSources = false;
+		
+		var toBeConnected = {};	// a map of new nodes to be connected
+		var toBeUpdated = {};	// a map of existing node to be updated with new group, streams
+		
+		if (this.oldSources && this.oldSources.length > 0) {
+			var streamIds = [];
+			var nodeIds = [];
+			
+			_.each(sources.streamConnections, function(source) {
+				streamIds.push(Util._streamId(source));
+				nodeIds.push(Util._nodeId(source));
+			});
+			
+			// remove from corresponding old sources not exist in new sources from data nodes
+			var stopStreamsByGroup = {};
+			_.each(this.oldSources, _.bind(function(oldSource) {
+				
+				if (!_.contains(streamIds, Util._streamId(oldSource)) || !_.contains(nodeIds, Util._nodeId(oldSource))) {
+					
+					console.log("[Subscriber] ", oldSource, " is no longer in the discovery results. Going to be removed.")
+					
+					_.each(this.nodes, _.bind(function(dataNode) {
+						
+						if (dataNode.hasStream(oldSource.group, oldSource.stream)) {
+							console.log("[Subscriber] Remove old stream ", oldSource, " from Data Node ", dataNode.id, " dataNode counter ", dataNode.counter);
+							dataNode.removeStream(oldSource.group, oldSource.stream, true);
+							if(stopStreamsByGroup[oldSource.group] === undefined){
+								stopStreamsByGroup[oldSource.group] = [];
+							}
+							stopStreamsByGroup[oldSource.group].push(oldSource.stream);
+						}
+						if(!_.isEmpty(stopStreamsByGroup)) {
+							console.log("[DataNode] Emitting consumption error after removing streams")
+							this.eventBus.emit(DataNode.Event.CONSUMPTION_ERROR, dataNode, stopStreamsByGroup);
+						}
+					}, this));
+				}
+				
+			}, this));
+			
+		}
+		
+		_.each(sources.streamConnections, _.bind(function(source) {	// for each source, we check if we are already consuming it
+			
+			/**
+			 * if we are not currently consuming a particular group & stream,
+			 * or if we are consuming it but the connection for that group & stream is having
+			 * problem, we will gather the group & stream by source id. 
+			 */
+			if (!this._exist(source) || 
+					// if there is error at the node currently consuming the stream
+					// and the connection returned by discoverer is different from the one
+					// currently use (which is causing error)
+					this._error(source) || this._hasUpdatedConnection(source)) {
+				
+				var nodeId = Util._nodeId(source);
+				
+				if (this._hasNode(nodeId)) {
+					if (toBeUpdated[nodeId] == undefined) toBeUpdated[nodeId] = [];
+					toBeUpdated[nodeId].push(source);
+				} else {
+					if (toBeConnected[nodeId] == undefined) toBeConnected[nodeId] = [];
+					toBeConnected[nodeId].push(source);
+				}
+			}
+			
+		}, this));
+		
+		// any group, and stream which are in these list should be removed from current
+		// data node first, before being added to new data node
+		this._connectUpdate(toBeConnected);
+		this._connectUpdate(toBeUpdated);
+		
+	},
+
+	_reconnectStreams : function(sources) {
+		var toBeConnected = {};	// a map of new nodes to be connected
+		var toBeUpdated = {};	// a map of existing node to be updated with new group, streams
+		_.each(this.discoverer.getSources().streamConnections, _.bind(function(source) {	// for each source, we check if we are already consuming it
+
+			/**
+			* if we are not currently consuming a particular group & stream,
+			* or if we are consuming it but the connection for that group & stream is having
+			* problem, we will gather the group & stream by source id. 
+			*/
+			if (!this._exist(source) || 
+				// if there is error at the node currently consuming the stream
+				// and the connection returned by discoverer is different from the one
+				// currently use (which is causing error)
+				(this._error(source) && this._hasUpdatedConnection(source))) {
+
+				console.log("Reconnecting to stopped stream ", source);
+				var nodeId = Util._nodeId(source);
+				if (this._hasNode(nodeId)) {
+					if (toBeUpdated[nodeId] == undefined) toBeUpdated[nodeId] = [];
+					toBeUpdated[nodeId].push(source);
+				} else {
+					if (toBeConnected[nodeId] == undefined) toBeConnected[nodeId] = [];
+					toBeConnected[nodeId].push(source);
+				}
+			}
+
+		}, this));
+		this._connectUpdate(toBeConnected);
+		this._connectUpdate(toBeUpdated);
+	},
+	
+	onNodeReconnectionError : function(dataNode, num) {
+		
+		if (num == this.config().exclusionThreshold) {
+			this.discoverer.excluding(dataNode.id);
+			this._updateSourcesOnNodeReconnectionError();
+			console.log("Excluding node ", dataNode.id, " from discovery");
+		}
+		
+	},
+	
+	_updateSourcesOnNodeReconnectionError : function(){
+		// Check if currentAvailableSources is not empty and not used.
+		if(this.hasUsedCurrentSource == false && _.isEmpty(this.currentAvailableSources) == false){
+			this._updateSources(this.currentAvailableSources);
+			this.hasUsedCurrentSources = true; // mark currentAvailableSources has used.
+		}
+	},
+	
+	onNodeStateChanged : function(dataNode, oldState) {
+		
+		// if the data node is destroyed or disconnected. Need to inform clients about the consumption error
+		// also mark those sources provided by that nodes as having error state
+		if (dataNode.state == DataNode.State.DESTROYED || dataNode.state == DataNode.State.DISCONNECTED) {
+							
+			for(var property in this.consuming){
+				if(this.consuming.hasOwnProperty(property)){
+					if(this.consuming[property].node.id == dataNode.id){
+						this.consuming[property].state = this.consumptionState.ERROR;
+					}
+				}
+			}
+			
+			var streamsByGroup = {};
+			
+			for(var property in dataNode.streams){
+				if(dataNode.streams.hasOwnProperty(property)){
+					streamsByGroup[property] = [];
+					for (streamInfo in dataNode.streams[property])
+						streamsByGroup[property].push(streamInfo.stream);
+				}
+			}
+			
+			if (!_.isEmpty(streamsByGroup)) {	// if streamsByGroup doesn't contain any thing, no need to emit event
+				console.log("[Subscriber] emitting consumption error due to data node failed.", dataNode.state);
+				this.dispatch(new Subscriber.Event.ConsumptionError(dataNode.client, streamsByGroup));
+			}
+		}
+		
+		if (oldState === DataNode.State.DISCONNECTED && dataNode.state === DataNode.State.CONNECTED) {
+			this.discoverer.removeExclusion(dataNode.id);
+			console.log("Node ", dataNode.id, " resume successfully. Remove it from exclusion.");
+		} else if (dataNode.state === DataNode.State.DESTROYED) {
+			// when it's destroyed, we are not going to reconnect to it automatically, so remove from exclusion
+			this.discoverer.removeExclusion(dataNode.id);
+			delete this.nodes[dataNode.id];
+			console.log("Node ", dataNode.id, " is destroyed and removed from subscriber!");
+			
+		}
+		
+	},
+
+	onStreamConsumption : function(dataNode, streamsByGroup){
+		this.dispatch(new Subscriber.Event.ConsumptionStart(dataNode.client, streamsByGroup));
+	},
+	
+	onStreamError : function(dataNode, streamsByGroup){
+		this.dispatch(new Subscriber.Event.ConsumptionError(dataNode.client, streamsByGroup));
+	},
+
+	onStreamStop : function(dataNode, streamsByGroup){
+		this.dispatch(new Subscriber.Event.ConsumptionStop(dataNode.client, streamsByGroup));
+	},
+	
+	_discovererError : function (err) {
+		
+	},
+	
+	/**
+	 * This method remove the group & stream which are going to be 
+	 * consumed in NEW DataNode or to be ADDED to other EXISTING
+	 * DataNodes from the current DataNodes they belong to (because current DataNodes
+	 * might be having connection error, which marks the respective group & streams
+	 * as error). Only when there is no one else consuming the stream, 
+	 * the stream can be registered into a new node. 
+	 * 
+	 * @param sources : a map from the nodeId of nodes to be added / updated to the group and stream. 
+	 */
+	_connectUpdate : function(sources) {
+		
+		_.each(sources, _.bind(function(connections, nodeId) {
+						
+			_.each(connections, _.bind(function(streamInfo) {
+				var streamId = Util._streamId(streamInfo);
+				
+				if (!this.consuming[streamId]) this.consuming[streamId] = new StreamConsumptionWrapper();
+				
+				var wrapper = this.consuming[streamId];
+				
+				var promise = Promise.resolve(0);
+				
+				// this will automatically destroy the node if there is no stream for it to consume
+				if (wrapper !== undefined && wrapper.node != undefined && wrapper.state == this.consumptionState.ERROR)
+					promise = wrapper.node.removeStream(streamInfo.group, streamInfo.stream, false);	// if consumption state is already error, no need to send deregistration
+				
+				promise.done = function(onFulfilled, onRejected) {
+					this.then(onFulfilled, onRejected)
+						.catch(function (reason) {
+							console.log("Promise Rejected: " + reason + "\n" + reason.stack);
+						});
+				};
+				
+				// this stream is going to be consume soon
+				wrapper.state = this.consumptionState.PENDING;
+				/**
+				 * when the promise is done
+				 * it means that there is no one consuming the group & stream
+				 * it's okay now that I'll start consuming it
+				 */
+				if (this.nodes[nodeId]) {	// if the node already existed
+					promise.done(_.bind(function(result) { 
+						
+						this.nodes[nodeId].consumeStream(streamInfo);
+						wrapper.node = this.nodes[nodeId];	// mark this so that no one else can consume this stream
+						
+					}, this));
+				} else {	// a new data node has to be created
+					
+					this.nodes[nodeId] = new DataNode(nodeId, this.clientId, this.ticket, streamInfo, this.eventBus, this.config());
+					promise.done(_.bind(function(result) {
+						wrapper.node = this.nodes[nodeId];	// mark this so that no one else can consume this stream
+						this.nodes[nodeId].connect();
+						
+					}, this));
+					
+				}
+				
+			}, this));
+			
+		}, this));
+		
+	},
+	
+	/**
+	 * Check if a group and stream existed
+	 * This means, the subscriber is consuming data for this group and stream from
+	 * a source.
+	 */
+	_exist : function(source) {
+		var streamId = Util._streamId(source);
+		
+		if (this.consuming[streamId]) return true;
+		
+		return false;
+	},
+	
+	/**
+	 * Check if a consuming group and stream is having any problem.
+	 * A consuming group and stream has problem if the DataNode for it is having 
+	 * connection problem.
+	 */
+	_error : function(source) {
+		var streamId = Util._streamId(source);
+		
+		if (this.consuming[streamId] && this.consuming[streamId].state == this.consumptionState.ERROR) return true;
+		
+		return false;
+	},
+	
+	/**
+	 * Check if a source returned by discoverer has newer connection
+	 * than the one being used
+	 */
+	_hasUpdatedConnection : function (source) {
+		var streamId = Util._streamId(source);
+		
+		if (!this.consuming[streamId] || !this.consuming[streamId].node || 
+				this.consuming[streamId].node.state == DataNode.State.DESTROYED) {
+			console.log("Has updated source");
+			
+			if (!this.consuming[streamId]) 
+				console.log("Because the source is not consuming");
+			else if (!this.consuming[streamId].node) 
+				console.log("Because the source is not consumed from a node");
+			else if (this.consuming[streamId].node == DataNode.State.DESTROYED) 
+				console.log("Because the source's node does not exist");
+			
+			return true;	// if this source is not in the consuming list or if it is, but there is no node for it
+		} else if (Util._nodeId(source) != this.consuming[streamId].node.id) {
+			console.log("Has updated source because the new node id is different from previous!");
+			return true;	// if the service id of the source is different from service id of the node
+		}
+		
+		return false;
+	},
+	
+	_hasNode : function(nodeId) {
+		return this.nodes[nodeId] !== undefined;
+	},
+	
+}, IEndPointEventSource);
+
+module.exports = exports = {
+	Subscriber : Subscriber,
+	SubscriberConfig : SubscriberConfig
+}